From 00678a12f0c2e490daf42118aee57ea2be2a5958 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:06:57 +0200 Subject: [PATCH] fix(deployments): retry transient GitHub token and clone failures Retry the installation token request on GitHub 5xx and connection errors, and report the HTTP status when GitHub sends no error message. Retry the repository import when SSH drops the connection (exit code 255), after stopping the interrupted clone and removing the partial checkout. Closes #8004 Co-Authored-By: Claude Opus 5.5 --- app/Jobs/ApplicationDeploymentJob.php | 34 ++++++- bootstrap/helpers/github.php | 13 ++- .../ApplicationDeploymentImportRetryTest.php | 89 +++++++++++++++++++ .../GithubAppInstallationTokenTest.php | 48 +++++++++- 4 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 tests/Feature/ApplicationDeploymentImportRetryTest.php diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index ec25b44b98..fe01e5b32f 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -3182,13 +3182,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private function clone_repository() { - $importCommands = $this->generate_git_import_commands(); $this->application_deployment_queue->addLogEntry("\n----------------------------------------"); $this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->commit}) to {$this->basedir}."); if ($this->pull_request_id !== 0) { $this->application_deployment_queue->addLogEntry("Checking out tag pull/{$this->pull_request_id}/head."); } - $this->execute_remote_command(...$this->gitCommandDefinitions($importCommands)); + $this->import_repository(); $this->create_workdir(); $this->execute_remote_command( [ @@ -3206,6 +3205,37 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * SSH exits with 255 when the connection drops, which can happen during a long clone. Git exits with + * 128 for its own errors, so only 255 is retried. The interrupted clone can still run in the helper + * container, so stop it and remove the partial checkout before cloning again. + */ + private function import_repository(): void + { + $maxAttempts = config('constants.ssh.max_retries'); + + for ($attempt = 1; ; $attempt++) { + try { + $this->execute_remote_command(...$this->gitCommandDefinitions($this->generate_git_import_commands())); + + return; + } catch (DeploymentException $e) { + if ($e->getCode() !== 255 || $attempt >= $maxAttempts) { + throw $e; + } + + $delay = $this->calculateRetryDelay($attempt - 1); + $this->application_deployment_queue->addLogEntry('Lost the SSH connection while importing the repository. Retrying in '.$delay.'s (attempt '.($attempt + 1)."/{$maxAttempts}).", 'stderr'); + Sleep::for($delay)->seconds(); + $this->execute_remote_command([ + executeInDocker($this->deployment_uuid, 'pkill git; rm -rf '.escapeshellarg($this->basedir)), + 'hidden' => true, + 'ignore_errors' => true, + ]); + } + } + } + private function generate_git_import_commands() { ['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands( diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.php index 48498c0135..edcbca88f6 100644 --- a/bootstrap/helpers/github.php +++ b/bootstrap/helpers/github.php @@ -4,6 +4,8 @@ use App\Models\GithubApp; use App\Models\PrivateKey; use Carbon\Carbon; use Carbon\CarbonImmutable; +use Illuminate\Http\Client\ConnectionException; +use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; @@ -195,13 +197,20 @@ function generateGithubToken(GithubApp $source, string $type) return match ($type) { 'jwt' => $jwt, 'installation' => (function () use ($source, $jwt) { + // GitHub sometimes answers with a 5xx page or drops the connection; a new token request is harmless. $response = Http::GitSource($source->api_url)->withHeaders([ 'Authorization' => "Bearer $jwt", 'Accept' => 'application/vnd.github.machine-man-preview+json', - ])->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); + ])->retry( + 3, + fn (int $attempt) => $attempt * 1000, + fn (Throwable $exception) => $exception instanceof ConnectionException + || ($exception instanceof RequestException && $exception->response->serverError()), + throw: false, + )->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); if (! $response->successful()) { - $error = data_get($response->json(), 'message', 'no error message found'); + $error = data_get($response->json(), 'message', "HTTP {$response->status()}"); if ($error === 'Not Found') { $error = 'Repository not found. Is it moved or deleted?'; } diff --git a/tests/Feature/ApplicationDeploymentImportRetryTest.php b/tests/Feature/ApplicationDeploymentImportRetryTest.php new file mode 100644 index 0000000000..e05bb97f1c --- /dev/null +++ b/tests/Feature/ApplicationDeploymentImportRetryTest.php @@ -0,0 +1,89 @@ +commands[] = $command[0]; + if (str_contains($command[0], 'git clone') && $exitCode = array_shift($this->importExitCodes)) { + throw new DeploymentException('Command execution failed', $exitCode); + } + } + } +} + +beforeEach(function () { + Sleep::fake(); + config()->set('constants.ssh.max_retries', 3); + + $application = Mockery::mock(Application::class)->makePartial(); + $application->shouldReceive('generateGitImportCommands')->andReturn([ + 'commands' => "docker exec deploy-uuid bash -c 'git clone repo /artifacts/deploy-uuid'", + 'branch' => 'main', + 'fullRepoUrl' => 'repo', + ]); + + $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial(); + $queue->shouldReceive('addLogEntry'); + + $this->job = new ImportRetryDeploymentJob; + foreach ([ + 'application' => $application, + 'application_deployment_queue' => $queue, + 'deployment_uuid' => 'deploy-uuid', + 'basedir' => '/artifacts/deploy-uuid', + 'pull_request_id' => 0, + 'commit' => 'HEAD', + ] as $property => $value) { + (new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($this->job, $value); + } +}); + +function importRepository(ImportRetryDeploymentJob $job): void +{ + (new ReflectionMethod(ApplicationDeploymentJob::class, 'import_repository'))->invoke($job); +} + +function cloneCommandCount(ImportRetryDeploymentJob $job): int +{ + return collect($job->commands)->filter(fn (string $command) => str_contains($command, 'git clone'))->count(); +} + +it('cleans up and clones again when the SSH connection drops during the import', function () { + $this->job->importExitCodes = [255]; + + importRepository($this->job); + + expect(cloneCommandCount($this->job))->toBe(2) + ->and($this->job->commands[1])->toBe(executeInDocker('deploy-uuid', "pkill git; rm -rf '/artifacts/deploy-uuid'")); +}); + +it('does not retry the import when git itself fails', function () { + $this->job->importExitCodes = [128]; + + expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class); + expect(cloneCommandCount($this->job))->toBe(1); +}); + +it('stops retrying the import after the configured SSH attempts', function () { + $this->job->importExitCodes = [255, 255, 255]; + + expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class); + expect(cloneCommandCount($this->job))->toBe(3); +}); diff --git a/tests/Feature/GithubAppInstallationTokenTest.php b/tests/Feature/GithubAppInstallationTokenTest.php index bb83df0636..9ea349c460 100644 --- a/tests/Feature/GithubAppInstallationTokenTest.php +++ b/tests/Feature/GithubAppInstallationTokenTest.php @@ -6,10 +6,13 @@ use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; uses(RefreshDatabase::class); beforeEach(function () { + Sleep::fake(); + $this->team = Team::factory()->create(); openssl_pkey_export(openssl_pkey_new([ @@ -36,24 +39,36 @@ beforeEach(function () { 'https://api.github.com/zen' => fn () => Http::response('Keep it logically awesome.', 200, [ 'Date' => now()->toRfc7231String(), ]), - 'https://api.github.com/app/installations/*/access_tokens' => Http::sequence() - ->push(['token' => 'first-token'], 201) - ->push(['token' => 'second-token'], 201), ]); }); +function fakeGithubInstallationTokenResponses(mixed $response): void +{ + Http::fake(['https://api.github.com/app/installations/*/access_tokens' => $response]); +} + +function fakeTwoGithubInstallationTokens(): void +{ + fakeGithubInstallationTokenResponses(Http::sequence() + ->push(['token' => 'first-token'], 201) + ->push(['token' => 'second-token'], 201)); +} + function githubInstallationTokenRequests(): int { return Http::recorded(fn (Request $request): bool => str_ends_with($request->url(), '/access_tokens'))->count(); } it('reuses the installation token of the same GitHub App instance', function () { + fakeTwoGithubInstallationTokens(); + expect(generateGithubInstallationToken($this->githubApp))->toBe('first-token') ->and(generateGithubInstallationToken($this->githubApp))->toBe('first-token') ->and(githubInstallationTokenRequests())->toBe(1); }); it('requests a new installation token after the in-memory token expires', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); $this->travel(51)->minutes(); @@ -63,6 +78,7 @@ it('requests a new installation token after the in-memory token expires', functi }); it('requests a new installation token when the GitHub App installation changes', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); $this->githubApp->installation_id = 11111; @@ -72,8 +88,34 @@ it('requests a new installation token when the GitHub App installation changes', }); it('does not share the installation token between GitHub App instances', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); expect(generateGithubInstallationToken(GithubApp::find($this->githubApp->id)))->toBe('second-token') ->and(githubInstallationTokenRequests())->toBe(2); }); + +it('retries the installation token request when GitHub returns a server error', function () { + fakeGithubInstallationTokenResponses(Http::sequence() + ->push('Bad Gateway', 502) + ->push(['token' => 'retried-token'], 201)); + + expect(generateGithubInstallationToken($this->githubApp))->toBe('retried-token') + ->and(githubInstallationTokenRequests())->toBe(2); +}); + +it('does not retry the installation token request when GitHub rejects it', function () { + fakeGithubInstallationTokenResponses(Http::response(['message' => 'Bad credentials'], 401)); + + expect(fn () => generateGithubInstallationToken($this->githubApp)) + ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: Bad credentials') + ->and(githubInstallationTokenRequests())->toBe(1); +}); + +it('reports the HTTP status when GitHub keeps failing without an error message', function () { + fakeGithubInstallationTokenResponses(Http::response('Service Unavailable', 503)); + + expect(fn () => generateGithubInstallationToken($this->githubApp)) + ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: HTTP 503') + ->and(githubInstallationTokenRequests())->toBe(3); +});