diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ad154d717..057fdd716c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Use PostgreSQL 16 in the Outline service template. New Outline versions need PostgreSQL 14 or later, so a new Outline service restarted in a loop on `postgres:12-alpine` with `syntax error at or near "TRIGGER"`. An existing Outline service keeps PostgreSQL 12. To upgrade it, follow the steps in the Outline service documentation ([#12345](https://github.com/coollabsio/coolify/issues/12345)) - Fix the LiteLLM service template for new services. The master key now starts with `sk-`, as LiteLLM requires: use `sk-` followed by the value of `SERVICE_PASSWORD_MASTERKEY` as the admin API key. The template also sets a separate `LITELLM_SALT_KEY`, so stored model credentials stay readable when you change the master key, and sets `STORE_MODEL_IN_DB` to `True` by default, so you can add models in the LiteLLM UI ([#10521](https://github.com/coollabsio/coolify/issues/10521)) - Send one request less to GitHub before each deployment and each list of repositories of a GitHub App. Before, Coolify always called `/zen` to compare its clock with GitHub before it requested an installation token, so a short network problem between the Coolify instance and GitHub failed the deployment with `cURL error 28: Connection timed out ... for https://api.github.com/zen`. Coolify now compares the clocks only when GitHub rejects a request, from the `Date` header of that response, and still shows "System time is out of sync with GitHub API time" ([#9785](https://github.com/coollabsio/coolify/issues/9785)) +- Retry GitHub API reads, such as the repository lookup before each deployment, up to three times when GitHub answers with a 5xx error or the connection fails. Before, one `502` or one timeout from GitHub failed the deployment. Writes, such as pull request comments, are not retried ([#9785](https://github.com/coollabsio/coolify/issues/9785)) - Keep an application that uses a server only as an additional destination when you delete that server with **Delete all resources** or with `DELETE /api/v1/servers/{uuid}?force=true`. Before, Coolify deleted the application on all its servers, also after **Promote** made another server its main server. Coolify now removes the application only from the deleted server and stops its containers there. A server that has only such applications can be deleted without **Delete all resources**. The delete confirmation lists the resources that it deletes and the applications that keep running on another server ## [v4.6.0] - 2026-11-02 diff --git a/bootstrap/helpers/github.php b/bootstrap/helpers/github.php index 1cc463ddbb..ddfff8c295 100644 --- a/bootstrap/helpers/github.php +++ b/bootstrap/helpers/github.php @@ -183,6 +183,15 @@ function assertGithubClockInSync(Response $response): void } } +/** + * GitHub sometimes answers with a 5xx page or drops the connection. A repeated read or token request is harmless. + */ +function isTransientGithubFailure(Throwable $exception): bool +{ + return $exception instanceof ConnectionException + || ($exception instanceof RequestException && $exception->response->serverError()); +} + function githubAppPrivateKey(GithubApp $source): ?PrivateKey { $privateKey = $source->privateKey; @@ -210,17 +219,10 @@ function generateGithubToken(GithubApp $source, string $type) return match ($type) { 'jwt' => $jwt, 'installation' => (function () use ($source, $jwt) { - // GitHub sometimes answers with a 5xx page or drops the connection; a new token request is harmless. $response = Http::GitSource($source->api_url)->withHeaders([ 'Authorization' => "Bearer $jwt", 'Accept' => 'application/vnd.github.machine-man-preview+json', - ])->retry( - 3, - fn (int $attempt) => $attempt * 1000, - fn (Throwable $exception) => $exception instanceof ConnectionException - || ($exception instanceof RequestException && $exception->response->serverError()), - throw: false, - )->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); + ])->retry(3, fn (int $attempt) => $attempt * 1000, isTransientGithubFailure(...), throw: false)->post("{$source->api_url}/app/installations/{$source->installation_id}/access_tokens"); if (! $response->successful()) { assertGithubClockInSync($response); @@ -257,15 +259,19 @@ function githubApi(?GithubApp $source, string $endpoint, string $method = 'get', throw new InvalidArgumentException("Unsupported source type: {$source->getMorphClass()}"); } - if ($source->is_public) { - $response = Http::GitHub($source->api_url)->$method($endpoint); + $request = $source->is_public + ? Http::GitHub($source->api_url) + : Http::GitHub($source->api_url, generateGithubInstallationToken($source)); + + // Only reads are retried: a repeated POST could create a second pull request comment. + if (strtolower($method) === 'get') { + $request = $request->retry(3, fn (int $attempt) => $attempt * 1000, isTransientGithubFailure(...), throw: false); + } + + if ($data && in_array(strtolower($method), ['post', 'patch', 'put'])) { + $response = $request->$method($endpoint, $data); } else { - $token = generateGithubInstallationToken($source); - if ($data && in_array(strtolower($method), ['post', 'patch', 'put'])) { - $response = Http::GitHub($source->api_url, $token)->$method($endpoint, $data); - } else { - $response = Http::GitHub($source->api_url, $token)->$method($endpoint); - } + $response = $request->$method($endpoint); } if (! $response->successful() && $throwError) { diff --git a/tests/Feature/GithubAppInstallationTokenTest.php b/tests/Feature/GithubAppInstallationTokenTest.php index 4811cd12be..78b9fc9016 100644 --- a/tests/Feature/GithubAppInstallationTokenTest.php +++ b/tests/Feature/GithubAppInstallationTokenTest.php @@ -133,3 +133,44 @@ it('reports the clock difference when GitHub rejects the token request of an out expect(fn () => generateGithubInstallationToken($this->githubApp)) ->toThrow(Exception::class, 'System time is out of sync with GitHub API time'); }); + +function githubRepositoryRequests(): int +{ + return Http::recorded(fn (Request $request): bool => str_contains($request->url(), '/repos/'))->count(); +} + +it('retries a GitHub API read when GitHub returns a server error', function () { + Http::fake([ + 'https://api.github.com/app/installations/*/access_tokens' => Http::response(['token' => 'only-token'], 201), + 'https://api.github.com/repos/*' => Http::sequence() + ->push('Bad Gateway', 502) + ->push(['id' => 42], 200), + ]); + + expect(data_get(githubApi($this->githubApp, 'repos/coollabsio/coolify'), 'data.id'))->toBe(42) + ->and(githubRepositoryRequests())->toBe(2) + ->and(githubInstallationTokenRequests())->toBe(1); +}); + +it('retries a GitHub API read after the connection retry of the HTTP client also times out', function () { + Http::fake([ + 'https://api.github.com/app/installations/*/access_tokens' => Http::response(['token' => 'only-token'], 201), + 'https://api.github.com/repos/*' => Http::sequence() + ->pushFailedConnection('cURL error 28: Connection timed out after 10002 milliseconds') + ->pushFailedConnection('cURL error 28: Connection timed out after 10002 milliseconds') + ->push(['id' => 42], 200), + ]); + + expect(data_get(githubApi($this->githubApp, 'repos/coollabsio/coolify'), 'data.id'))->toBe(42); +}); + +it('does not retry a GitHub API write when GitHub returns a server error', function () { + Http::fake([ + 'https://api.github.com/app/installations/*/access_tokens' => Http::response(['token' => 'only-token'], 201), + 'https://api.github.com/repos/*' => Http::response('Bad Gateway', 502), + ]); + + expect(fn () => githubApi($this->githubApp, 'repos/coollabsio/coolify/issues/1/comments', 'post', ['body' => 'Deployed'])) + ->toThrow(Exception::class, 'GitHub API call failed') + ->and(githubRepositoryRequests())->toBe(1); +});