diff --git a/app/Exceptions/CloudProviderApiException.php b/app/Exceptions/CloudProviderApiException.php new file mode 100644 index 0000000000..287d4aafd9 --- /dev/null +++ b/app/Exceptions/CloudProviderApiException.php @@ -0,0 +1,73 @@ + */ + public readonly array $invalidFields; + + /** + * The error code and field names come from the provider, so only values that look like identifiers are kept. + * + * @param array $invalidFields + */ + public function __construct( + public readonly string $provider, + string $providerMessage, + public readonly int $status, + mixed $errorCode = null, + array $invalidFields = [], + ) { + parent::__construct("{$provider} API error: {$providerMessage}", $status); + + $errorCode = is_string($errorCode) ? strtolower($errorCode) : null; + $this->errorCode = $errorCode !== null && preg_match('/^[a-z_]{1,64}$/', $errorCode) ? $errorCode : null; + $this->invalidFields = array_values(array_filter( + $invalidFields, + fn ($name) => is_string($name) && preg_match('/^[a-z0-9_.\[\]]{1,64}$/i', $name) + )); + } + + /** + * A message that is safe to return to API clients. It is built only from the + * HTTP status, the error code, and field names, never from the provider's free-text message. + */ + public function safeMessage(): string + { + return match (true) { + $this->errorCode === 'resource_limit_exceeded' => "Your {$this->provider} account has reached a resource limit, for example the server limit. Remove unused resources or ask {$this->provider} for a higher limit.", + $this->errorCode === 'resource_unavailable' => 'The selected server type, image, or location is not available now. Select a different location or server type.', + $this->errorCode === 'uniqueness_error' => "A resource with the same name already exists in your {$this->provider} account, for example a server or an SSH key. Use a different name.", + $this->errorCode === 'maintenance', $this->status === 423, $this->status >= 500 => "{$this->provider} cannot process the request now. Try again later.", + $this->status === 401 => "The {$this->provider} API token is invalid or expired. Update the cloud provider token in Coolify.", + $this->status === 403 => "The {$this->provider} API token does not have permission for this request. Use a token with read and write access.", + $this->status === 404 => "{$this->provider} could not find a requested resource, for example the location, server type, image, or SSH key.", + $this->status === 409 => "{$this->provider} rejected the request because of a conflict with an existing resource.", + $this->status === 422 => "{$this->provider} rejected the request as invalid. Check the selected values and your account limits.".($this->invalidFields !== [] ? ' Invalid fields: '.implode(', ', $this->invalidFields).'.' : ''), + default => "{$this->provider} returned HTTP {$this->status}".($this->errorCode !== null ? " (error code: {$this->errorCode})" : '').'.', + }; + } + + /** + * The API response for a failed provider request: 502, because the upstream API rejected it. + */ + public function toJsonResponse(string $failureMessage): JsonResponse + { + return response()->json([ + 'message' => $failureMessage.' '.$this->safeMessage(), + 'provider_status' => $this->status, + 'provider_error_code' => $this->errorCode, + ], 502); + } +} diff --git a/app/Http/Controllers/Api/DigitalOceanController.php b/app/Http/Controllers/Api/DigitalOceanController.php index 427a805278..7c7dd6e00b 100644 --- a/app/Http/Controllers/Api/DigitalOceanController.php +++ b/app/Http/Controllers/Api/DigitalOceanController.php @@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api; use App\Actions\Server\ValidateServer; use App\Enums\ProxyTypes; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use App\Http\Controllers\Api\Concerns\RequiresDeployAbility; use App\Http\Controllers\Controller; @@ -210,6 +211,7 @@ class DigitalOceanController extends Controller new OA\Response(response: 403, description: 'The token needs the deploy permission for a cloud-init script or extra SSH keys.'), new OA\Response(response: 422, description: 'Validation failed.'), new OA\Response(response: 429, description: 'DigitalOcean rate limit exceeded.'), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function createServer(Request $request): JsonResponse @@ -395,12 +397,13 @@ class DigitalOceanController extends Controller } return $response; - } catch (\Throwable $e) { + } catch (CloudProviderApiException $e) { $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server); - logger()->error('Failed to create DigitalOcean server', [ - 'error' => $e->getMessage(), - ]); + return $e->toJsonResponse('Failed to create DigitalOcean server.'); + } catch (\Throwable $e) { + $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server); + report($e); return response()->json(['message' => 'Failed to create DigitalOcean server.'], 500); } diff --git a/app/Http/Controllers/Api/HetznerController.php b/app/Http/Controllers/Api/HetznerController.php index ca0ec99116..509558a00f 100644 --- a/app/Http/Controllers/Api/HetznerController.php +++ b/app/Http/Controllers/Api/HetznerController.php @@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api; use App\Actions\Server\ValidateServer; use App\Enums\ProxyTypes; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use App\Http\Controllers\Api\Concerns\RequiresDeployAbility; use App\Http\Controllers\Controller; @@ -90,6 +91,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function locations(Request $request) @@ -127,7 +129,11 @@ class HetznerController extends Controller $locations = $hetznerService->getLocations(); return response()->json($locations); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner locations.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner locations.'], 500); } } @@ -212,6 +218,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function serverTypes(Request $request) @@ -249,7 +256,11 @@ class HetznerController extends Controller $serverTypes = $hetznerService->getServerTypes(); return response()->json($serverTypes); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner server types.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner server types.'], 500); } } @@ -312,6 +323,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function images(Request $request) @@ -362,7 +374,11 @@ class HetznerController extends Controller }); return response()->json(array_values($filtered)); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner images.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner images.'], 500); } } @@ -422,6 +438,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function sshKeys(Request $request) @@ -459,7 +476,11 @@ class HetznerController extends Controller $sshKeys = $hetznerService->getSshKeys(); return response()->json($sshKeys); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner SSH keys.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner SSH keys.'], 500); } } @@ -517,6 +538,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function firewalls(Request $request) @@ -553,7 +575,11 @@ class HetznerController extends Controller $hetznerService = new HetznerService($token->token); return response()->json($hetznerService->getFirewalls()); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner firewalls.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner firewalls.'], 500); } } @@ -612,6 +638,7 @@ class HetznerController extends Controller response: 404, ref: '#/components/responses/404', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function networks(Request $request) @@ -648,7 +675,11 @@ class HetznerController extends Controller $hetznerService = new HetznerService($token->token); return response()->json($hetznerService->getNetworks()); + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to fetch Hetzner networks.'); } catch (\Throwable $e) { + report($e); + return response()->json(['message' => 'Failed to fetch Hetzner networks.'], 500); } } @@ -728,6 +759,7 @@ class HetznerController extends Controller response: 429, ref: '#/components/responses/429', ), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function createServer(Request $request) @@ -1012,7 +1044,18 @@ class HetznerController extends Controller } return $response; + } catch (CloudProviderApiException $e) { + return $e->toJsonResponse('Failed to create Hetzner server.'); } catch (\Throwable $e) { + report($e); + if (isset($hetznerService, $hetznerServer['id'])) { + try { + $hetznerService->deleteServer((int) $hetznerServer['id']); + } catch (\Throwable $cleanupError) { + report($cleanupError); + } + } + return response()->json(['message' => 'Failed to create Hetzner server.'], 500); } } diff --git a/app/Http/Controllers/Api/OpenApi.php b/app/Http/Controllers/Api/OpenApi.php index 1279442b28..4af7e503bd 100644 --- a/app/Http/Controllers/Api/OpenApi.php +++ b/app/Http/Controllers/Api/OpenApi.php @@ -198,6 +198,17 @@ use OpenApi\Attributes as OA; new OA\Property(property: 'message', type: 'string', example: 'Rate limit exceeded. Please try again later.'), ] )), + new OA\Response( + response: 502, + description: 'The cloud provider API rejected the request.', + content: new OA\JsonContent( + type: 'object', + properties: [ + new OA\Property(property: 'message', type: 'string', example: 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.'), + new OA\Property(property: 'provider_status', type: 'integer', description: 'The HTTP status from the cloud provider API.', example: 403), + new OA\Property(property: 'provider_error_code', type: 'string', nullable: true, description: 'The error code from the cloud provider API, if it has one.', example: 'forbidden'), + ] + )), ], )] class OpenApi diff --git a/app/Http/Controllers/Api/VultrController.php b/app/Http/Controllers/Api/VultrController.php index 31aa2b4bac..86601b8e4d 100644 --- a/app/Http/Controllers/Api/VultrController.php +++ b/app/Http/Controllers/Api/VultrController.php @@ -4,6 +4,7 @@ namespace App\Http\Controllers\Api; use App\Actions\Server\ValidateServer; use App\Enums\ProxyTypes; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use App\Http\Controllers\Api\Concerns\RequiresDeployAbility; use App\Http\Controllers\Controller; @@ -193,6 +194,7 @@ class VultrController extends Controller new OA\Response(response: 403, description: 'The token needs the deploy permission for a cloud-init script or extra SSH keys.'), new OA\Response(response: 422, description: 'Validation failed.'), new OA\Response(response: 429, description: 'Vultr API rate limit exceeded.'), + new OA\Response(response: 502, ref: '#/components/responses/502'), ] )] public function createServer(Request $request): JsonResponse @@ -403,12 +405,13 @@ class VultrController extends Controller } return $response; - } catch (\Throwable $e) { + } catch (CloudProviderApiException $e) { $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server); - logger()->error('Failed to create Vultr server', [ - 'error' => $e->getMessage(), - ]); + return $e->toJsonResponse('Failed to create Vultr server.'); + } catch (\Throwable $e) { + $this->deleteUntrackedInstance($vultrService, $vultrInstanceId, $server); + report($e); return response()->json(['message' => 'Failed to create Vultr server.'], 500); } diff --git a/app/Livewire/Server/New/ByHetzner.php b/app/Livewire/Server/New/ByHetzner.php index 868b9e44b3..fbc6d93a9b 100644 --- a/app/Livewire/Server/New/ByHetzner.php +++ b/app/Livewire/Server/New/ByHetzner.php @@ -3,6 +3,7 @@ namespace App\Livewire\Server\New; use App\Enums\ProxyTypes; +use App\Exceptions\CloudProviderApiException; use App\Models\CloudInitScript; use App\Models\CloudProviderToken; use App\Models\PrivateKey; @@ -12,7 +13,6 @@ use App\Rules\ValidCloudInitYaml; use App\Rules\ValidHostname; use App\Services\HetznerService; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; -use Illuminate\Http\Client\RequestException; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; use Illuminate\Validation\ValidationException; @@ -358,22 +358,13 @@ class ByHetzner extends Component $this->loading_data = false; } catch (\Throwable $e) { $this->loading_data = false; - $this->provider_data_error = $this->providerDataErrorMessage('Hetzner', $e, 'error.message'); + $this->provider_data_error = $e instanceof CloudProviderApiException + ? $e->getMessage() + : 'Hetzner API error: '.$e->getMessage(); $this->dispatch('error', $this->provider_data_error); } } - private function providerDataErrorMessage(string $providerName, \Throwable $e, string $jsonMessageKey): string - { - $details = $e->getMessage(); - - if ($e instanceof RequestException && $e->response) { - $details = data_get($e->response->json(), $jsonMessageKey) ?: $e->response->body() ?: $details; - } - - return "{$providerName} API error: {$details}"; - } - private function getCpuVendorInfo(array $serverType): ?string { $name = strtolower($serverType['name'] ?? ''); @@ -766,20 +757,31 @@ class ByHetzner extends Component if (! isset($e->errors()['server'])) { throw $e; } - if (isset($hetznerService, $hetznerServer['id'])) { - try { - $hetznerService->deleteServer((int) $hetznerServer['id']); - } catch (\Throwable $cleanupError) { - report($cleanupError); - } - } + $this->deleteUntrackedHetznerServer($hetznerService ?? null, $hetznerServer['id'] ?? null); return $this->dispatch('error', 'You have reached the server limit for your subscription.'); } catch (\Throwable $e) { + if (! isset($server)) { + $this->deleteUntrackedHetznerServer($hetznerService ?? null, $hetznerServer['id'] ?? null); + } + return handleError($e, $this); } } + private function deleteUntrackedHetznerServer(?HetznerService $hetznerService, ?int $hetznerServerId): void + { + if (! $hetznerService || ! $hetznerServerId) { + return; + } + + try { + $hetznerService->deleteServer($hetznerServerId); + } catch (\Throwable $e) { + report($e); + } + } + public function render() { return view('livewire.server.new.by-hetzner'); diff --git a/app/Services/DigitalOceanService.php b/app/Services/DigitalOceanService.php index da1af942f4..423fc4f0dc 100644 --- a/app/Services/DigitalOceanService.php +++ b/app/Services/DigitalOceanService.php @@ -2,6 +2,7 @@ namespace App\Services; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Http; @@ -45,7 +46,7 @@ class DigitalOceanService ); } - throw new \Exception('DigitalOcean API error: '.$response->json('message', 'Unknown error'), $response->status()); + throw new CloudProviderApiException('DigitalOcean', $response->json('message', 'Unknown error'), $response->status(), $response->json('id')); } return $response->json() ?? []; diff --git a/app/Services/HetznerService.php b/app/Services/HetznerService.php index c9675a24c8..faeeee654f 100644 --- a/app/Services/HetznerService.php +++ b/app/Services/HetznerService.php @@ -2,6 +2,7 @@ namespace App\Services; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use Illuminate\Http\Client\RequestException; use Illuminate\Support\Facades\Http; @@ -44,7 +45,17 @@ class HetznerService // Exponential backoff for other retriable errors: 100ms, 200ms, 400ms return $attempt * 100; - }) + }, function (\Exception $exception) use ($method) { + if ($exception instanceof RequestException && $exception->response->status() === 429) { + return true; + } + + // A POST that failed with a 5xx or a timeout can still have created the resource, so only idempotent requests are sent again. + // Client errors such as 401, 403, or 422 do not change when the request is sent again. + $isServerError = ! $exception instanceof RequestException || $exception->response->status() >= 500; + + return $isServerError && in_array(strtolower($method), ['get', 'delete'], true); + }, throw: false) ->{$method}($this->baseUrl.$endpoint, $data); if (! $response->successful()) { @@ -61,7 +72,13 @@ class HetznerService ); } - throw new \Exception('Hetzner API error: '.$response->json('error.message', 'Unknown error')); + throw new CloudProviderApiException( + 'Hetzner', + $response->json('error.message', 'Unknown error'), + $response->status(), + $response->json('error.code'), + collect($response->json('error.details.fields', []))->pluck('name')->all(), + ); } return $response->json(); diff --git a/app/Services/VultrService.php b/app/Services/VultrService.php index 347b7e3fbb..ced66a66f7 100644 --- a/app/Services/VultrService.php +++ b/app/Services/VultrService.php @@ -2,6 +2,7 @@ namespace App\Services; +use App\Exceptions\CloudProviderApiException; use App\Exceptions\RateLimitException; use Illuminate\Support\Facades\Http; @@ -28,7 +29,7 @@ class VultrService ); } - throw new \Exception('Vultr API error: '.$response->json('error', 'Unknown error'), $response->status()); + throw new CloudProviderApiException('Vultr', $response->json('error', 'Unknown error'), $response->status()); } return $response->json() ?? []; diff --git a/openapi.json b/openapi.json index 6b1ce726f5..2640e3d628 100644 --- a/openapi.json +++ b/openapi.json @@ -11529,6 +11529,9 @@ }, "429": { "description": "DigitalOcean rate limit exceeded." + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12567,6 +12570,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12678,6 +12684,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12758,6 +12767,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12829,6 +12841,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12894,6 +12909,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -12962,6 +12980,9 @@ }, "404": { "$ref": "#\/components\/responses\/404" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -13122,6 +13143,9 @@ }, "429": { "$ref": "#\/components\/responses\/429" + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -24009,6 +24033,9 @@ }, "429": { "description": "Vultr API rate limit exceeded." + }, + "502": { + "$ref": "#\/components\/responses\/502" } }, "security": [ @@ -26425,6 +26452,35 @@ } } } + }, + "502": { + "description": "The cloud provider API rejected the request.", + "content": { + "application\/json": { + "schema": { + "properties": { + "message": { + "type": "string", + "example": "Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access." + }, + "provider_status": { + "description": "The HTTP status from the cloud provider API.", + "type": "integer", + "example": 403 + }, + "provider_error_code": { + "description": "The error code from the cloud provider API, if it has one.", + "type": [ + "string", + "null" + ], + "example": "forbidden" + } + }, + "type": "object" + } + } + } } }, "securitySchemes": { diff --git a/openapi.yaml b/openapi.yaml index 85f5c49369..2340bff7d9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -7536,6 +7536,8 @@ paths: description: 'Validation failed.' '429': description: 'DigitalOcean rate limit exceeded.' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8117,6 +8119,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8157,6 +8161,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8197,6 +8203,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8237,6 +8245,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8277,6 +8287,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8317,6 +8329,8 @@ paths: $ref: '#/components/responses/401' '404': $ref: '#/components/responses/404' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -8423,6 +8437,8 @@ paths: $ref: '#/components/responses/422' '429': $ref: '#/components/responses/429' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -15197,6 +15213,8 @@ paths: description: 'Validation failed.' '429': description: 'Vultr API rate limit exceeded.' + '502': + $ref: '#/components/responses/502' security: - bearerAuth: [] @@ -16935,6 +16953,26 @@ components: type: string example: 'Rate limit exceeded. Please try again later.' type: object + '502': + description: 'The cloud provider API rejected the request.' + content: + application/json: + schema: + properties: + message: + type: string + example: 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.' + provider_status: + description: 'The HTTP status from the cloud provider API.' + type: integer + example: 403 + provider_error_code: + description: 'The error code from the cloud provider API, if it has one.' + type: + - string + - 'null' + example: forbidden + type: object securitySchemes: bearerAuth: type: http diff --git a/tests/Feature/Api/HetznerApiTest.php b/tests/Feature/Api/HetznerApiTest.php index 4246eb7878..2916004e71 100644 --- a/tests/Feature/Api/HetznerApiTest.php +++ b/tests/Feature/Api/HetznerApiTest.php @@ -808,8 +808,142 @@ describe('POST /api/v1/servers/hetzner', function () { }); }); +describe('POST /api/v1/servers/hetzner upstream errors', function () { + beforeEach(function () { + $this->createServerRequest = fn () => $this->withHeaders([ + 'Authorization' => 'Bearer '.$this->bearerToken, + 'Content-Type' => 'application/json', + ])->postJson('/api/v1/servers/hetzner', [ + 'cloud_provider_token_id' => $this->hetznerToken->uuid, + 'location' => 'nbg1', + 'server_type' => 'cx11', + 'image' => 15512617, + 'name' => 'test-server', + 'private_key_uuid' => $this->privateKey->uuid, + 'enable_ipv4' => true, + 'enable_ipv6' => true, + ]); + + $this->fakeHetzner = fn (Closure $createServerResponse) => Http::fake(function (HttpRequest $request) use ($createServerResponse) { + if ($request->method() === 'GET' && str_starts_with($request->url(), 'https://api.hetzner.cloud/v1/ssh_keys')) { + return Http::response(['ssh_keys' => [], 'meta' => ['pagination' => ['next_page' => null]]], 200); + } + if ($request->method() === 'POST' && $request->url() === 'https://api.hetzner.cloud/v1/ssh_keys') { + return Http::response(['ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd']], 201); + } + if ($request->method() === 'POST' && $request->url() === 'https://api.hetzner.cloud/v1/servers') { + return $createServerResponse(); + } + if ($request->method() === 'DELETE' && $request->url() === 'https://api.hetzner.cloud/v1/servers/456') { + return Http::response([], 200); + } + + return Http::response([], 404); + }); + }); + + test('explains a read-only token without leaking the Hetzner message and does not retry', function () { + ($this->fakeHetzner)(fn () => Http::response([ + 'error' => ['code' => 'forbidden', 'message' => 'INTERNAL_LEAK_TOKEN_abc'], + ], 403)); + + $response = ($this->createServerRequest)(); + + $response->assertStatus(502); + $response->assertExactJson([ + 'message' => 'Failed to create Hetzner server. The Hetzner API token does not have permission for this request. Use a token with read and write access.', + 'provider_status' => 403, + 'provider_error_code' => 'forbidden', + ]); + expect($response->getContent())->not->toContain('INTERNAL_LEAK_TOKEN_abc'); + Http::assertSentCount(3); + $this->assertDatabaseMissing('servers', ['name' => 'test-server']); + }); + + test('lists the invalid fields Hetzner reports', function () { + ($this->fakeHetzner)(fn () => Http::response([ + 'error' => [ + 'code' => 'invalid_input', + 'message' => 'invalid input in field server_type', + 'details' => ['fields' => [['name' => 'server_type'], ['name' => '