diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index ec25b44b98..fe01e5b32f 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -3182,13 +3182,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private function clone_repository() { - $importCommands = $this->generate_git_import_commands(); $this->application_deployment_queue->addLogEntry("\n----------------------------------------"); $this->application_deployment_queue->addLogEntry("Importing {$this->customRepository}:{$this->application->git_branch} (commit sha {$this->commit}) to {$this->basedir}."); if ($this->pull_request_id !== 0) { $this->application_deployment_queue->addLogEntry("Checking out tag pull/{$this->pull_request_id}/head."); } - $this->execute_remote_command(...$this->gitCommandDefinitions($importCommands)); + $this->import_repository(); $this->create_workdir(); $this->execute_remote_command( [ @@ -3206,6 +3205,37 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue } } + /** + * SSH exits with 255 when the connection drops, which can happen during a long clone. Git exits with + * 128 for its own errors, so only 255 is retried. The interrupted clone can still run in the helper + * container, so stop it and remove the partial checkout before cloning again. + */ + private function import_repository(): void + { + $maxAttempts = config('constants.ssh.max_retries'); + + for ($attempt = 1; ; $attempt++) { + try { + $this->execute_remote_command(...$this->gitCommandDefinitions($this->generate_git_import_commands())); + + return; + } catch (DeploymentException $e) { + if ($e->getCode() !== 255 || $attempt >= $maxAttempts) { + throw $e; + } + + $delay = $this->calculateRetryDelay($attempt - 1); + $this->application_deployment_queue->addLogEntry('Lost the SSH connection while importing the repository. Retrying in '.$delay.'s (attempt '.($attempt + 1)."/{$maxAttempts}).", 'stderr'); + Sleep::for($delay)->seconds(); + $this->execute_remote_command([ + executeInDocker($this->deployment_uuid, 'pkill git; rm -rf '.escapeshellarg($this->basedir)), + 'hidden' => true, + 'ignore_errors' => true, + ]); + } + } + } + private function generate_git_import_commands() { ['commands' => $commands, 'branch' => $this->branch, 'fullRepoUrl' => $this->fullRepoUrl] = $this->application->generateGitImportCommands( diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.php index 48498c0135..edcbca88f6 100644 --- a/bootstrap/helpers/github.php +++ b/bootstrap/helpers/github.php @@ -4,6 +4,8 @@ use App\Models\GithubApp; use App\Models\PrivateKey; use Carbon\Carbon; use Carbon\CarbonImmutable; +use Illuminate\Http\Client\ConnectionException; +use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Http; use Illuminate\Support\Str; @@ -195,13 +197,20 @@ function generateGithubToken(GithubApp $source, string $type) return match ($type) { 'jwt' => $jwt, 'installation' => (function () use ($source, $jwt) { + // GitHub sometimes answers with a 5xx page or drops the connection; a new token request is harmless. $response = Http::GitSource($source->api_url)->withHeaders([ 'Authorization' => "Bearer $jwt", 'Accept' => 'application/vnd.github.machine-man-preview+json', - ])->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); + ])->retry( + 3, + fn (int $attempt) => $attempt * 1000, + fn (Throwable $exception) => $exception instanceof ConnectionException + || ($exception instanceof RequestException && $exception->response->serverError()), + throw: false, + )->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); if (! $response->successful()) { - $error = data_get($response->json(), 'message', 'no error message found'); + $error = data_get($response->json(), 'message', "HTTP {$response->status()}"); if ($error === 'Not Found') { $error = 'Repository not found. Is it moved or deleted?'; } diff --git a/tests/Feature/ApplicationDeploymentImportRetryTest.php b/tests/Feature/ApplicationDeploymentImportRetryTest.php new file mode 100644 index 0000000000..e05bb97f1c --- /dev/null +++ b/tests/Feature/ApplicationDeploymentImportRetryTest.php @@ -0,0 +1,89 @@ +commands[] = $command[0]; + if (str_contains($command[0], 'git clone') && $exitCode = array_shift($this->importExitCodes)) { + throw new DeploymentException('Command execution failed', $exitCode); + } + } + } +} + +beforeEach(function () { + Sleep::fake(); + config()->set('constants.ssh.max_retries', 3); + + $application = Mockery::mock(Application::class)->makePartial(); + $application->shouldReceive('generateGitImportCommands')->andReturn([ + 'commands' => "docker exec deploy-uuid bash -c 'git clone repo /artifacts/deploy-uuid'", + 'branch' => 'main', + 'fullRepoUrl' => 'repo', + ]); + + $queue = Mockery::mock(ApplicationDeploymentQueue::class)->makePartial(); + $queue->shouldReceive('addLogEntry'); + + $this->job = new ImportRetryDeploymentJob; + foreach ([ + 'application' => $application, + 'application_deployment_queue' => $queue, + 'deployment_uuid' => 'deploy-uuid', + 'basedir' => '/artifacts/deploy-uuid', + 'pull_request_id' => 0, + 'commit' => 'HEAD', + ] as $property => $value) { + (new ReflectionProperty(ApplicationDeploymentJob::class, $property))->setValue($this->job, $value); + } +}); + +function importRepository(ImportRetryDeploymentJob $job): void +{ + (new ReflectionMethod(ApplicationDeploymentJob::class, 'import_repository'))->invoke($job); +} + +function cloneCommandCount(ImportRetryDeploymentJob $job): int +{ + return collect($job->commands)->filter(fn (string $command) => str_contains($command, 'git clone'))->count(); +} + +it('cleans up and clones again when the SSH connection drops during the import', function () { + $this->job->importExitCodes = [255]; + + importRepository($this->job); + + expect(cloneCommandCount($this->job))->toBe(2) + ->and($this->job->commands[1])->toBe(executeInDocker('deploy-uuid', "pkill git; rm -rf '/artifacts/deploy-uuid'")); +}); + +it('does not retry the import when git itself fails', function () { + $this->job->importExitCodes = [128]; + + expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class); + expect(cloneCommandCount($this->job))->toBe(1); +}); + +it('stops retrying the import after the configured SSH attempts', function () { + $this->job->importExitCodes = [255, 255, 255]; + + expect(fn () => importRepository($this->job))->toThrow(DeploymentException::class); + expect(cloneCommandCount($this->job))->toBe(3); +}); diff --git a/tests/Feature/GithubAppInstallationTokenTest.php b/tests/Feature/GithubAppInstallationTokenTest.php index bb83df0636..9ea349c460 100644 --- a/tests/Feature/GithubAppInstallationTokenTest.php +++ b/tests/Feature/GithubAppInstallationTokenTest.php @@ -6,10 +6,13 @@ use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Http\Client\Request; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; uses(RefreshDatabase::class); beforeEach(function () { + Sleep::fake(); + $this->team = Team::factory()->create(); openssl_pkey_export(openssl_pkey_new([ @@ -36,24 +39,36 @@ beforeEach(function () { 'https://api.github.com/zen' => fn () => Http::response('Keep it logically awesome.', 200, [ 'Date' => now()->toRfc7231String(), ]), - 'https://api.github.com/app/installations/*/access_tokens' => Http::sequence() - ->push(['token' => 'first-token'], 201) - ->push(['token' => 'second-token'], 201), ]); }); +function fakeGithubInstallationTokenResponses(mixed $response): void +{ + Http::fake(['https://api.github.com/app/installations/*/access_tokens' => $response]); +} + +function fakeTwoGithubInstallationTokens(): void +{ + fakeGithubInstallationTokenResponses(Http::sequence() + ->push(['token' => 'first-token'], 201) + ->push(['token' => 'second-token'], 201)); +} + function githubInstallationTokenRequests(): int { return Http::recorded(fn (Request $request): bool => str_ends_with($request->url(), '/access_tokens'))->count(); } it('reuses the installation token of the same GitHub App instance', function () { + fakeTwoGithubInstallationTokens(); + expect(generateGithubInstallationToken($this->githubApp))->toBe('first-token') ->and(generateGithubInstallationToken($this->githubApp))->toBe('first-token') ->and(githubInstallationTokenRequests())->toBe(1); }); it('requests a new installation token after the in-memory token expires', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); $this->travel(51)->minutes(); @@ -63,6 +78,7 @@ it('requests a new installation token after the in-memory token expires', functi }); it('requests a new installation token when the GitHub App installation changes', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); $this->githubApp->installation_id = 11111; @@ -72,8 +88,34 @@ it('requests a new installation token when the GitHub App installation changes', }); it('does not share the installation token between GitHub App instances', function () { + fakeTwoGithubInstallationTokens(); generateGithubInstallationToken($this->githubApp); expect(generateGithubInstallationToken(GithubApp::find($this->githubApp->id)))->toBe('second-token') ->and(githubInstallationTokenRequests())->toBe(2); }); + +it('retries the installation token request when GitHub returns a server error', function () { + fakeGithubInstallationTokenResponses(Http::sequence() + ->push('Bad Gateway', 502) + ->push(['token' => 'retried-token'], 201)); + + expect(generateGithubInstallationToken($this->githubApp))->toBe('retried-token') + ->and(githubInstallationTokenRequests())->toBe(2); +}); + +it('does not retry the installation token request when GitHub rejects it', function () { + fakeGithubInstallationTokenResponses(Http::response(['message' => 'Bad credentials'], 401)); + + expect(fn () => generateGithubInstallationToken($this->githubApp)) + ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: Bad credentials') + ->and(githubInstallationTokenRequests())->toBe(1); +}); + +it('reports the HTTP status when GitHub keeps failing without an error message', function () { + fakeGithubInstallationTokenResponses(Http::response('Service Unavailable', 503)); + + expect(fn () => generateGithubInstallationToken($this->githubApp)) + ->toThrow(RuntimeException::class, 'Failed to get installation token for Private GitHub App with error: HTTP 503') + ->and(githubInstallationTokenRequests())->toBe(3); +});