fix(deployments): prevent Git fetches from hanging (#12321)

app/Jobs/ApplicationDeploymentJob.php

@@ -3171,11 +3171,7 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue
 
     private function gitLsRemoteCommand(string $lsRemoteRef, ?string $identityFile = null): string
     {
-        $sshCommand = "ssh -o ConnectTimeout=30 -p {$this->customPort} -o Port={$this->customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
-
-        if ($identityFile !== null) {
-            $sshCommand .= " -i {$identityFile} -o IdentitiesOnly=yes";
-        }
+        $sshCommand = gitSshCommand($this->customPort, $identityFile);
 
         return 'GIT_SSH_COMMAND="'.$sshCommand.'" git ls-remote '.escapeshellarg($this->fullRepoUrl).' '.escapeshellarg($lsRemoteRef);
     }

app/Models/Application.php

@@ -1587,7 +1587,7 @@ class Application extends BaseModel
             // If shallow clone is enabled and we need a specific commit,
             // we need to fetch that specific commit with depth=1
             if ($isShallowCloneEnabled) {
-                $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
+                $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && echo 'Fetching the commit.' && {$sshCommand} {$gitCommand} fetch --depth=1 origin {$escapedCommit} && {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
             } else {
                 $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} -c advice.detachedHead=false checkout {$escapedCommit} >/dev/null 2>&1";
             }
@@ -1601,10 +1601,12 @@ class Application extends BaseModel
             }
             // Add shallow submodules flag if shallow clone is enabled
             $submoduleFlags = $isShallowCloneEnabled ? '--depth=1' : '';
-            $git_clone_command = "{$git_clone_command} {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";
+            $git_clone_command = "{$git_clone_command} echo 'Updating submodules.' && {$gitCommand} submodule sync && {$sshCommand} {$gitCommand} submodule update --init --recursive {$submoduleFlags}; fi";
         }
+        // `git lfs pull` opens an SSH session even without LFS files, which can hang on hosts
+        // without LFS over SSH (Azure DevOps). `git lfs ls-files` is offline.
         if ($this->settings->is_git_lfs_enabled && ! $onlyCheckout) {
-            $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && {$sshCommand} {$gitCommand} lfs pull";
+            $git_clone_command = "{$git_clone_command} && cd {$escapedBaseDir} && if {$gitCommand} lfs ls-files -n | grep -q .; then echo 'Downloading Git LFS objects.' && {$sshCommand} {$gitCommand} lfs pull; fi";
         }
 
         return $git_clone_command;
@@ -1714,7 +1716,8 @@ class Application extends BaseModel
                     $private_key = base64_encode($private_key);
                     $gitlabPort = $oauthSource->custom_port ?? 22;
                     $escapedCustomRepository = str_replace("'", "'\\''", $customRepository);
-                    $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";
+                    $sshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);
+                    $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";
 
                     $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
 
@@ -1759,7 +1762,8 @@ class Application extends BaseModel
             // When used with executeInDocker (which uses bash -c '...'), we need to escape for bash context
             // Replace ' with '\'' to safely escape within single-quoted bash strings
             $escapedCustomRepository = str_replace("'", "'\\''", $customRepository);
-            $base_command = "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes\" {$base_command} '{$escapedCustomRepository}'";
+            $sshCommand = gitSshCommand($customPort, $customSshKeyLocation);
+            $base_command = "GIT_SSH_COMMAND=\"{$sshCommand}\" {$base_command} '{$escapedCustomRepository}'";
 
             $commands = $this->gitSshKeySetupCommands($deployment_uuid, $private_key, $exec_in_docker);
 
@@ -1993,7 +1997,7 @@ class Application extends BaseModel
                     $private_key = base64_encode($private_key);
                     $gitlabPort = $oauthSource->custom_port ?? 22;
                     $escapedCustomRepository = escapeshellarg($customRepository);
-                    $gitlabSshCommand = "ssh -o ConnectTimeout=30 -p {$gitlabPort} -o Port={$gitlabPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";
+                    $gitlabSshCommand = gitSshCommand($gitlabPort, $customSshKeyLocation);
                     $gitlabGitSshCommand = "GIT_SSH_COMMAND=\"{$gitlabSshCommand}\"";
                     $git_clone_command_base = "{$gitlabGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
                     if ($only_checkout) {
@@ -2054,7 +2058,7 @@ class Application extends BaseModel
             }
             $private_key = base64_encode($private_key);
             $escapedCustomRepository = escapeshellarg($customRepository);
-            $deployKeySshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$customSshKeyLocation} -o IdentitiesOnly=yes";
+            $deployKeySshCommand = gitSshCommand($customPort, $customSshKeyLocation);
             $deployKeyGitSshCommand = "GIT_SSH_COMMAND=\"{$deployKeySshCommand}\"";
             $git_clone_command_base = "{$deployKeyGitSshCommand} {$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
             if ($only_checkout) {
@@ -2108,7 +2112,7 @@ class Application extends BaseModel
             $escapedCustomRepository = escapeshellarg($customRepository);
             $git_clone_command = "{$git_clone_command} {$escapedCustomRepository} {$escapedBaseDir}";
             $git_clone_command = $this->setGitImportSettings($deployment_uuid, $git_clone_command, public: true, commit: $commit, baseDir: $baseDir, onlyCheckout: $only_checkout);
-            $otherSshCommand = "ssh -o ConnectTimeout=30 -p {$customPort} -o Port={$customPort} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa";
+            $otherSshCommand = gitSshCommand($customPort, '/root/.ssh/id_rsa', identitiesOnly: false);
 
             if ($pull_request_id !== 0) {
                 if ($git_type === 'gitlab') {

bootstrap/helpers/applications.php

@@ -553,3 +553,18 @@ function clone_application(Application $source, $destination, array $overrides =
 
     return $newApplication;
 }
+
+/**
+ * The SSH command that git uses to reach a repository.
+ * The keepalive options stop a stalled connection after about one minute; without them git waits until the deployment timeout.
+ */
+function gitSshCommand(int|string $port, ?string $identityFile = null, bool $identitiesOnly = true): string
+{
+    $command = "ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p {$port} -o Port={$port} -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null";
+
+    if ($identityFile !== null) {
+        $command .= " -i {$identityFile}".($identitiesOnly ? ' -o IdentitiesOnly=yes' : '');
+    }
+
+    return $command;
+}

tests/Unit/DeployKeyDedicatedPathTest.php

@@ -262,8 +262,8 @@ it('uses the per-deployment deploy key for pull request fetches', function () us
 
     $result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);
 
-    expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");
-    expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
+    expectCommandListToContain($result['commands'], "GIT_SSH_COMMAND=\"ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {$keyPath} -o IdentitiesOnly=yes\" git fetch origin pull/123/head:pr-123-coolify");
+    expectCommandListNotToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
     expectPrivateKeyMaterializationCommandsSkipLogging($result['commands']);
 });
 
@@ -282,6 +282,6 @@ it('does not force a missing per-deployment key for other repository pull reques
 
     $result = $application->generateGitImportCommands('test-deployment-uuid', 123, 'github', false);
 
-    expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
+    expectCommandListToContain($result['commands'], 'GIT_SSH_COMMAND="ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa" git fetch origin pull/123/head:pr-123-coolify');
     expectCommandListNotToContain($result['commands'], $keyPath);
 });

tests/Unit/GitImportStepsTest.php

@@ -0,0 +1,84 @@
+<?php
+
+use App\Models\Application;
+use App\Models\ApplicationSetting;
+use Symfony\Component\Process\Process;
+
+/**
+ * Runs the generated import command in bash with a stub git that records its calls.
+ * The stub prints $lfsFiles for `git lfs ls-files -n`.
+ *
+ * @return array{0: int, 1: list<string>, 2: list<string>}
+ */
+function runImportCommandWithStubGit(string $lfsFiles, bool $hasSubmodules = false): array
+{
+    $dir = sys_get_temp_dir().'/coolify-import-'.bin2hex(random_bytes(4));
+    mkdir("{$dir}/bin", 0755, true);
+    file_put_contents("{$dir}/bin/git", <<<SH
+        #!/bin/sh
+        echo "\$*" >> {$dir}/git.log
+        if [ "\$*" = "lfs ls-files -n" ]; then printf '%s' '{$lfsFiles}'; fi
+        SH);
+    chmod("{$dir}/bin/git", 0755);
+    if ($hasSubmodules) {
+        touch("{$dir}/.gitmodules");
+    }
+
+    $application = new Application;
+    $application->forceFill(['uuid' => 'test-app-uuid', 'git_commit_sha' => 'abc123']);
+    $settings = new ApplicationSetting;
+    $settings->is_git_shallow_clone_enabled = true;
+    $settings->is_git_submodules_enabled = true;
+    $settings->is_git_lfs_enabled = true;
+    $application->setRelation('settings', $settings);
+
+    $command = $application->setGitImportSettings(
+        deployment_uuid: 'test-uuid',
+        git_clone_command: 'true',
+        gitSshCommand: 'ssh -o ConnectTimeout=30 -i /root/.ssh/id_rsa',
+        baseDir: $dir,
+    );
+
+    $process = Process::fromShellCommandline($command, env: ['PATH' => "{$dir}/bin:".getenv('PATH')]);
+    $process->run();
+    $calls = file_exists("{$dir}/git.log") ? file("{$dir}/git.log", FILE_IGNORE_NEW_LINES) : [];
+
+    exec('rm -rf '.escapeshellarg($dir));
+
+    return [$process->getExitCode(), $calls, explode("\n", trim($process->getOutput()))];
+}
+
+test('git lfs pull is skipped when the checkout has no LFS files', function () {
+    [$exitCode, $calls, $output] = runImportCommandWithStubGit('');
+
+    expect($exitCode)->toBe(0)
+        ->and($calls)->toBe([
+            'fetch --depth=1 origin abc123',
+            '-c advice.detachedHead=false checkout abc123',
+            'lfs ls-files -n',
+        ])
+        ->and($output)->toBe(['Fetching the commit.']);
+});
+
+test('each network step after the clone prints a log line before it runs', function () {
+    [$exitCode, $calls, $output] = runImportCommandWithStubGit('assets/logo.psd', hasSubmodules: true);
+
+    expect($exitCode)->toBe(0)
+        ->and($calls)->toBe([
+            'fetch --depth=1 origin abc123',
+            '-c advice.detachedHead=false checkout abc123',
+            'submodule sync',
+            'submodule update --init --recursive --depth=1',
+            'lfs ls-files -n',
+            'lfs pull',
+        ])
+        ->and($output)->toBe(['Fetching the commit.', 'Updating submodules.', 'Downloading Git LFS objects.']);
+});
+
+test('git ssh command stops a stalled connection instead of waiting for the deployment timeout', function () {
+    expect(gitSshCommand(2222, '/root/.ssh/key'))->toBe(
+        'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 2222 -o Port=2222 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/key -o IdentitiesOnly=yes'
+    )
+        ->and(gitSshCommand(22))->not->toContain(' -i ')
+        ->and(gitSshCommand(22, '/root/.ssh/id_rsa', identitiesOnly: false))->toEndWith('-i /root/.ssh/id_rsa');
+});

tests/Unit/GitSubmoduleCredentialTest.php

@@ -156,7 +156,7 @@ describe('Git submodule credential propagation', function () {
             exec_in_docker: false,
         );
 
-        $sshCommand = 'ssh -o ConnectTimeout=30 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';
+        $sshCommand = 'ssh -o ConnectTimeout=30 -o ServerAliveInterval=15 -o ServerAliveCountMax=4 -p 22 -o Port=22 -o LogLevel=ERROR -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i /root/.ssh/id_rsa_coolify_test-uuid -o IdentitiesOnly=yes';
 
         // Commands are a list of ['command' => ..., 'hidden' => ...] entries; assert on the joined shell script
         $commands = collect($result['commands'])