diff --git a/app/Helpers/SshRetryHandler.php b/app/Helpers/SshRetryHandler.php index aaaf4252a8..45aaf6f9fa 100644 --- a/app/Helpers/SshRetryHandler.php +++ b/app/Helpers/SshRetryHandler.php @@ -27,8 +27,8 @@ class SshRetryHandler /** * Convenience static method for retry execution */ - public static function retry(callable $callback, array $context = [], bool $throwError = true) + public static function retry(callable $callback, array $context = [], bool $throwError = true, ?int $maxAttempts = null) { - return self::instance()->executeWithSshRetry($callback, $context, $throwError); + return self::instance()->executeWithSshRetry($callback, $context, $throwError, $maxAttempts); } } diff --git a/app/Http/Controllers/Api/DestinationsController.php b/app/Http/Controllers/Api/DestinationsController.php index 9991df2793..a7d0c3c45e 100644 --- a/app/Http/Controllers/Api/DestinationsController.php +++ b/app/Http/Controllers/Api/DestinationsController.php @@ -284,6 +284,10 @@ class DestinationsController extends Controller private function existingServerNetworkResponse(Server $server, string $network): ?JsonResponse { + if (! $server->isFunctional()) { + return response()->json(['message' => 'Server is not functional.'], 400); + } + try { $output = instant_remote_process(["docker network ls --format '{{.Name}}'"], $server); } catch (\Throwable) { diff --git a/app/Http/Controllers/Api/ServerLogDrainsController.php b/app/Http/Controllers/Api/ServerLogDrainsController.php index af5aaa1dfd..a5da9edb89 100644 --- a/app/Http/Controllers/Api/ServerLogDrainsController.php +++ b/app/Http/Controllers/Api/ServerLogDrainsController.php @@ -274,10 +274,12 @@ class ServerLogDrainsController extends Controller $server->refresh(); // Match Livewire instantSave: start or stop the drain service after settings change. - if ($server->isLogDrainEnabled()) { - StartLogDrain::dispatch($server); - } else { - StopLogDrain::dispatch($server); + if ($server->isFunctional()) { + if ($server->isLogDrainEnabled()) { + StartLogDrain::dispatch($server); + } else { + StopLogDrain::dispatch($server); + } } auditLog('api.server.log_drains.updated', [ diff --git a/app/Http/Controllers/Api/ServerProxyController.php b/app/Http/Controllers/Api/ServerProxyController.php index 9861831123..dc5f1c73b0 100644 --- a/app/Http/Controllers/Api/ServerProxyController.php +++ b/app/Http/Controllers/Api/ServerProxyController.php @@ -389,6 +389,10 @@ class ServerProxyController extends Controller ], 422); } + if (! $server->isFunctional()) { + return response()->json(['message' => 'Server is not functional.'], 400); + } + SaveProxyConfiguration::run($server, $configuration); auditLog('api.server.proxy.configuration_saved', [ @@ -443,6 +447,10 @@ class ServerProxyController extends Controller $this->authorize('manageProxy', $server); + if (! $server->isFunctional()) { + return response()->json(['message' => 'Server is not functional.'], 400); + } + RestartProxyJob::dispatch($server); auditLog('api.server.proxy.restarted', [ diff --git a/app/Http/Controllers/Api/ServersController.php b/app/Http/Controllers/Api/ServersController.php index 6e1cc58c06..9be3a088b0 100644 --- a/app/Http/Controllers/Api/ServersController.php +++ b/app/Http/Controllers/Api/ServersController.php @@ -603,6 +603,9 @@ class ServersController extends Controller if ($foundServer) { return response()->json(['message' => 'A server with this IP/Domain is already in use.'], 400); } + if ($request->instant_validate) { + ModelsServer::throttleValidation($teamId, $request->ip); + } $proxyType = $request->proxy_type ? str($request->proxy_type)->upper() : ProxyTypes::TRAEFIK->value; @@ -772,6 +775,9 @@ class ServersController extends Controller return response()->json(['message' => 'Server not found.'], 404); } $this->authorize('update', $server); + if ($request->instant_validate) { + ModelsServer::throttleValidation($teamId, $request->ip ?? $server->ip); + } if ($request->proxy_type) { $validProxyTypes = collect(ProxyTypes::cases())->map(function ($proxyType) { return str($proxyType->value)->lower(); @@ -1137,6 +1143,7 @@ class ServersController extends Controller if ($install && ($response = $this->deployAbilityForbiddenResponse($request, 'Installing the server runs commands on it and starts the proxy, so it needs a token with the deploy permission.'))) { return $response; } + ModelsServer::throttleValidation($teamId, $server->ip); if ($install) { ValidateAndInstallServerJob::dispatch($server); } else { diff --git a/app/Livewire/Server/PrivateKey/Show.php b/app/Livewire/Server/PrivateKey/Show.php index 97eb760d2c..0673ac2aad 100644 --- a/app/Livewire/Server/PrivateKey/Show.php +++ b/app/Livewire/Server/PrivateKey/Show.php @@ -5,6 +5,7 @@ namespace App\Livewire\Server\PrivateKey; use App\Models\PrivateKey; use App\Models\Server; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; +use Illuminate\Http\Exceptions\ThrottleRequestsException; use Illuminate\Support\Facades\DB; use Livewire\Component; @@ -44,8 +45,13 @@ class Show extends Component return; } + $this->authorize('update', $this->server); + try { + Server::throttleValidation($this->server->team_id, $this->server->ip); + } catch (ThrottleRequestsException $e) { + return handleError($e, $this); + } try { - $this->authorize('update', $this->server); DB::transaction(function () use ($ownedPrivateKey) { $this->server->privateKey()->associate($ownedPrivateKey); $this->server->save(); @@ -95,6 +101,7 @@ class Show extends Component public function checkConnection() { try { + Server::throttleValidation($this->server->team_id, $this->server->ip); ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection(); if ($uptime) { $this->dispatch('success', 'Server is reachable.'); diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php index 23363cd9d7..42facdc9f9 100644 --- a/app/Livewire/Server/Show.php +++ b/app/Livewire/Server/Show.php @@ -442,6 +442,7 @@ class Show extends Component try { $this->authorize('update', $this->server); $this->syncData(true); + Server::throttleValidation($this->server->team_id, $this->server->ip); ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection(); if ($uptime) { $this->dispatch('success', 'Server is reachable.'); diff --git a/app/Livewire/Server/ValidateAndInstall.php b/app/Livewire/Server/ValidateAndInstall.php index 407f5f31d2..27a7c580e1 100644 --- a/app/Livewire/Server/ValidateAndInstall.php +++ b/app/Livewire/Server/ValidateAndInstall.php @@ -8,6 +8,7 @@ use App\Events\ServerValidated; use App\Jobs\CheckAndStartSentinelJob; use App\Models\Server; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; +use Livewire\Attributes\Locked; use Livewire\Component; class ValidateAndInstall extends Component @@ -22,6 +23,7 @@ class ValidateAndInstall extends Component public bool $install = true; + #[Locked] public $uptime = null; public $supported_os_type = null; @@ -113,6 +115,7 @@ class ValidateAndInstall extends Component { try { $this->authorize('update', $this->server); + Server::throttleValidation($this->server->team_id, $this->server->ip); if ($this->server->vultr_instance_id) { $status = $this->server->refreshVultrState(); $this->server->refresh(); @@ -177,6 +180,10 @@ class ValidateAndInstall extends Component { $this->authorize('update', $this->server); + if (! $this->uptime) { + return; + } + $this->supported_os_type = $this->server->validateOS(); if (! $this->supported_os_type) { $this->error = 'Server OS type is not supported. Please install Docker manually before continuing: documentation.'; @@ -193,6 +200,10 @@ class ValidateAndInstall extends Component { $this->authorize('update', $this->server); + if (! $this->uptime) { + return; + } + $validationResult = $this->server->validatePrerequisites(); $this->prerequisites_installed = $validationResult['success']; if (! $validationResult['success']) { @@ -233,6 +244,10 @@ class ValidateAndInstall extends Component { $this->authorize('update', $this->server); + if (! $this->uptime) { + return; + } + $this->docker_installed = $this->server->validateDockerEngine(); $this->docker_compose_installed = $this->server->validateDockerCompose(); if (! $this->docker_installed || ! $this->docker_compose_installed) { @@ -271,6 +286,10 @@ class ValidateAndInstall extends Component { $this->authorize('update', $this->server); + if (! $this->uptime) { + return; + } + if ($this->server->isSwarm()) { $swarmInstalled = $this->server->validateDockerSwarm(); if ($swarmInstalled) { diff --git a/app/Livewire/Settings/Advanced.php b/app/Livewire/Settings/Advanced.php index 86060ddd3f..17f9efc36d 100644 --- a/app/Livewire/Settings/Advanced.php +++ b/app/Livewire/Settings/Advanced.php @@ -66,6 +66,12 @@ class Advanced extends Component public string $file_download_storage = 'stream'; + public int $server_validation_rate_limit_seconds = 600; + + public int $server_validation_max_failures_per_ip = 10; + + public int $server_validation_max_failures_per_team = 30; + public function rules() { return [ @@ -85,6 +91,9 @@ class Advanced extends Component 'webhook_allow_localhost' => 'boolean', 'domain_connect_private_key' => 'nullable|string', 'image_cdn_url' => 'nullable|url|max:255', + 'server_validation_rate_limit_seconds' => 'required|integer|min:60|max:86400', + 'server_validation_max_failures_per_ip' => 'required|integer|min:1|max:1000', + 'server_validation_max_failures_per_team' => 'required|integer|min:1|max:10000', ]; } @@ -114,6 +123,9 @@ class Advanced extends Component ? 's3:'.$this->settings->avatar_s3_storage_id : 'local'; $this->image_cdn_url = $this->settings->image_cdn_url; + $this->server_validation_rate_limit_seconds = $this->settings->server_validation_rate_limit_seconds; + $this->server_validation_max_failures_per_ip = $this->settings->server_validation_max_failures_per_ip; + $this->server_validation_max_failures_per_team = $this->settings->server_validation_max_failures_per_team; $this->avatar_storage_options = [ ['value' => 'local', 'label' => 'Local storage'], ...S3Storage::query() @@ -200,6 +212,12 @@ class Advanced extends Component return; } + if (isCloud()) { + $this->settings->server_validation_rate_limit_seconds = $this->server_validation_rate_limit_seconds; + $this->settings->server_validation_max_failures_per_ip = $this->server_validation_max_failures_per_ip; + $this->settings->server_validation_max_failures_per_team = $this->server_validation_max_failures_per_team; + } + if (isCloud() && filled($this->domain_connect_private_key)) { $this->settings->domain_connect_private_key = $this->normalizeDomainConnectPrivateKey($this->domain_connect_private_key); $this->domain_connect_private_key = null; diff --git a/app/Models/InstanceSettings.php b/app/Models/InstanceSettings.php index 8726a9f0f6..fbc9e7c099 100644 --- a/app/Models/InstanceSettings.php +++ b/app/Models/InstanceSettings.php @@ -105,6 +105,9 @@ class InstanceSettings extends Model 'webhook_allow_localhost' => 'boolean', 'is_dashboard_force_https_enabled' => 'boolean', 'is_passkey_enabled' => 'boolean', + 'server_validation_rate_limit_seconds' => 'integer', + 'server_validation_max_failures_per_ip' => 'integer', + 'server_validation_max_failures_per_team' => 'integer', ]; protected static function booted(): void diff --git a/app/Models/Server.php b/app/Models/Server.php index ede3c00b8a..6e82272a1c 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -34,11 +34,13 @@ use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Database\Eloquent\Relations\MorphMany; use Illuminate\Database\Eloquent\SoftDeletes; +use Illuminate\Http\Exceptions\ThrottleRequestsException; use Illuminate\Support\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; +use Illuminate\Support\Facades\RateLimiter; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use Illuminate\Support\Stringable; @@ -1407,6 +1409,64 @@ $siteAddress { return ! $this->isTransferredAway(); } + /** + * On Coolify Cloud, limit how often SSH validations can run against one IP and from one team. + * The window and the maximums are server_validation_* instance settings. + * Without a limit, users can make Cloud try SSH logins on hosts that they do not own. + * A successful connection gives its attempt back, so only failed logins use up the limit. + * + * @throws ThrottleRequestsException + */ + public static function throttleValidation(int $teamId, string $ip): void + { + if (! isCloud()) { + return; + } + + $limits = self::validationRateLimits($teamId, $ip); + foreach ($limits as $key => $maxAttempts) { + if (RateLimiter::tooManyAttempts($key, $maxAttempts)) { + $seconds = RateLimiter::availableIn($key); + + throw new ThrottleRequestsException("Too many server validations. Please try again in {$seconds} seconds.", headers: ['Retry-After' => $seconds]); + } + } + foreach (array_keys($limits) as $key) { + RateLimiter::hit($key, self::validationRateLimitSeconds()); + } + } + + private static function validationRateLimitSeconds(): int + { + return instanceSettings()->server_validation_rate_limit_seconds; + } + + /** + * @return array rate limiter key => max attempts + */ + private static function validationRateLimits(int $teamId, string $ip): array + { + $settings = instanceSettings(); + + return [ + 'server-validation:ip:'.strtolower(self::normalizeIp($ip)) => $settings->server_validation_max_failures_per_ip, + 'server-validation:team:'.$teamId => $settings->server_validation_max_failures_per_team, + ]; + } + + private function refundValidationAttempt(): void + { + if (! isCloud()) { + return; + } + + foreach (array_keys(self::validationRateLimits($this->team_id, $this->ip)) as $key) { + if (RateLimiter::attempts($key) > 0) { + RateLimiter::decrement($key, self::validationRateLimitSeconds()); + } + } + } + public function forceEnableServer() { if ($this->isTransferredAway()) { @@ -2333,7 +2393,9 @@ $siteAddress { return ['uptime' => false, 'error' => 'Server skipped.']; } try { - instant_remote_process(['ls /'], $this, disableMultiplexing: true); + // One attempt only: retries would repeat failed logins on a host that may not belong to the user. + instant_remote_process(['ls /'], $this, disableMultiplexing: true, maxAttempts: 1); + $this->refundValidationAttempt(); if ($this->settings->is_reachable === false) { $this->settings->is_reachable = true; $this->settings->save(); diff --git a/app/Traits/SshRetryable.php b/app/Traits/SshRetryable.php index 537965df12..b275c1e471 100644 --- a/app/Traits/SshRetryable.php +++ b/app/Traits/SshRetryable.php @@ -90,11 +90,12 @@ trait SshRetryable * @param callable $callback The operation to execute * @param array $context Context for logging (server, command, etc.) * @param bool $throwError Whether to throw error on final failure + * @param int|null $maxAttempts Attempts in total, defaults to constants.ssh.max_retries * @return mixed The result from the callback */ - protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true) + protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true, ?int $maxAttempts = null) { - $maxRetries = config('constants.ssh.max_retries'); + $maxRetries = $maxAttempts ?? config('constants.ssh.max_retries'); $lastError = null; $lastErrorMessage = ''; // Randomly fail the command with a key exchange error for testing diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 877d790266..e5dd73de46 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -216,7 +216,7 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $ ); } -function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false): ?string +function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false, ?int $maxAttempts = null): ?string { $command = $command instanceof Collection ? $command->toArray() : $command; @@ -248,7 +248,8 @@ function instant_remote_process(Collection|array $command, Server $server, bool 'command_preview' => substr($command_string, 0, 100), 'function' => 'instant_remote_process', ], - $throwError + $throwError, + $maxAttempts ); } diff --git a/database/migrations/2026_10_23_090005_add_server_validation_limits_to_instance_settings_table.php b/database/migrations/2026_10_23_090005_add_server_validation_limits_to_instance_settings_table.php new file mode 100644 index 0000000000..50a4fbf797 --- /dev/null +++ b/database/migrations/2026_10_23_090005_add_server_validation_limits_to_instance_settings_table.php @@ -0,0 +1,34 @@ +unsignedInteger('server_validation_rate_limit_seconds')->default(600); + $table->unsignedInteger('server_validation_max_failures_per_ip')->default(10); + $table->unsignedInteger('server_validation_max_failures_per_team')->default(30); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('instance_settings', function (Blueprint $table) { + $table->dropColumn([ + 'server_validation_rate_limit_seconds', + 'server_validation_max_failures_per_ip', + 'server_validation_max_failures_per_team', + ]); + }); + } +}; diff --git a/resources/views/livewire/settings/advanced.blade.php b/resources/views/livewire/settings/advanced.blade.php index 39534ff215..68b5b383c7 100644 --- a/resources/views/livewire/settings/advanced.blade.php +++ b/resources/views/livewire/settings/advanced.blade.php @@ -115,6 +115,20 @@ @endif + +
+ + + +
+
@endif
diff --git a/tests/Feature/Api/DestinationsApiTest.php b/tests/Feature/Api/DestinationsApiTest.php index 6fb87584e6..674ddef037 100644 --- a/tests/Feature/Api/DestinationsApiTest.php +++ b/tests/Feature/Api/DestinationsApiTest.php @@ -44,6 +44,7 @@ beforeEach(function () { 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, ]); $this->destination = StandaloneDocker::where('server_id', $this->server->id)->first(); + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); }); function destinationsApiHeaders(string $bearerToken): array @@ -130,6 +131,18 @@ describe('POST /api/v1/servers/{server_uuid}/destinations', function () { ->exists())->toBeTrue(); }); + test('does not connect to a server that is not validated', function () { + $this->server->settings->update(['is_reachable' => false, 'is_usable' => false]); + + $this->withHeaders(destinationsApiHeaders($this->bearerToken)) + ->postJson("/api/v1/servers/{$this->server->uuid}/destinations", ['network' => 'api-standalone-network']) + ->assertStatus(400) + ->assertJsonPath('message', 'Server is not functional.'); + + Process::assertNothingRan(); + expect(StandaloneDocker::where('network', 'api-standalone-network')->exists())->toBeFalse(); + }); + test('requires a write token', function () { $readOnlyToken = destinationsApiToken($this->user, $this->team, ['read']); diff --git a/tests/Feature/Api/ServerDeployAbilityTest.php b/tests/Feature/Api/ServerDeployAbilityTest.php index 085b3bd506..041796f5f6 100644 --- a/tests/Feature/Api/ServerDeployAbilityTest.php +++ b/tests/Feature/Api/ServerDeployAbilityTest.php @@ -41,6 +41,7 @@ beforeEach(function () { ]); $this->server->proxy->set('type', ProxyTypes::TRAEFIK->value); $this->server->save(); + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); }); function serverDeployAbilityHeaders(User $user, int $teamId, array $abilities): array diff --git a/tests/Feature/Api/ServerProxyApiTest.php b/tests/Feature/Api/ServerProxyApiTest.php index 9a18bb8cf1..2900b48c84 100644 --- a/tests/Feature/Api/ServerProxyApiTest.php +++ b/tests/Feature/Api/ServerProxyApiTest.php @@ -216,6 +216,7 @@ test('PATCH /api/v1/servers/{uuid}/proxy can change proxy type asynchronously', }); test('PUT /api/v1/servers/{uuid}/proxy/configuration saves base64 configuration via action', function () { + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); $compose = "services:\n traefik:\n image: traefik:v3.5\n"; SaveProxyConfiguration::shouldRun() @@ -241,6 +242,7 @@ test('PUT /api/v1/servers/{uuid}/proxy/configuration rejects missing configurati }); test('PUT /api/v1/servers/{uuid}/proxy/configuration rejects command injection without mutation', function () { + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); $originalProxy = $this->server->proxy->toArray(); $configuration = "services:\n traefik:\n ports:\n - '`id>/tmp/pwned`:443'\n"; @@ -283,6 +285,7 @@ test('PUT /api/v1/servers/{uuid}/proxy/configuration requires the deploy permiss }); test('PUT /api/v1/servers/{uuid}/proxy/configuration saves with write and deploy permissions', function () { + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); $deployToken = serverProxyApiToken($this->user, $this->team, ['write', 'deploy']); SaveProxyConfiguration::shouldRun()->once(); @@ -318,6 +321,7 @@ test('POST /api/v1/servers/{uuid}/proxy/restart requires the deploy permission', }); test('POST /api/v1/servers/{uuid}/proxy/restart works with the deploy permission', function () { + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); Queue::fake(); $deployToken = serverProxyApiToken($this->user, $this->team, ['deploy']); @@ -329,6 +333,7 @@ test('POST /api/v1/servers/{uuid}/proxy/restart works with the deploy permission }); test('POST /api/v1/servers/{uuid}/proxy/restart queues RestartProxyJob', function () { + $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]); Queue::fake(); $this->withHeaders(serverProxyApiHeaders($this->bearerToken)) @@ -342,6 +347,25 @@ test('POST /api/v1/servers/{uuid}/proxy/restart queues RestartProxyJob', functio ); }); +test('proxy configuration and restart endpoints do not connect to a server that is not validated', function () { + Queue::fake(); + SaveProxyConfiguration::shouldNotRun(); + + $this->withHeaders(serverProxyApiHeaders($this->bearerToken)) + ->putJson("/api/v1/servers/{$this->server->uuid}/proxy/configuration", [ + 'configuration' => base64_encode("services:\n traefik:\n image: traefik:v3.5\n"), + ]) + ->assertStatus(400) + ->assertJsonPath('message', 'Server is not functional.'); + + $this->withHeaders(serverProxyApiHeaders($this->bearerToken)) + ->postJson("/api/v1/servers/{$this->server->uuid}/proxy/restart") + ->assertStatus(400) + ->assertJsonPath('message', 'Server is not functional.'); + + Queue::assertNotPushed(RestartProxyJob::class); +}); + test('proxy endpoints require authentication', function () { $this->getJson("/api/v1/servers/{$this->server->uuid}/proxy")->assertUnauthorized(); $this->patchJson("/api/v1/servers/{$this->server->uuid}/proxy", ['redirect_enabled' => false])->assertUnauthorized(); diff --git a/tests/Feature/Api/ServerSubsystemsApiTest.php b/tests/Feature/Api/ServerSubsystemsApiTest.php index a131bf22ab..712534538a 100644 --- a/tests/Feature/Api/ServerSubsystemsApiTest.php +++ b/tests/Feature/Api/ServerSubsystemsApiTest.php @@ -1,6 +1,8 @@ and($settings->logdrain_axiom_api_key)->toBe('axiom-key-123'); }); + test('PATCH saves log drain settings without connecting to a server that is not validated', function () { + Queue::fake(); + + $this->withHeaders(serverSubsystemsHeaders()) + ->patchJson("/api/v1/servers/{$this->server->uuid}/log-drains", [ + 'is_logdrain_axiom_enabled' => false, + ]) + ->assertOk(); + + StopLogDrain::assertNotPushed(); + StartLogDrain::assertNotPushed(); + }); + test('PATCH requires the deploy permission to change the custom log drain config', function (string $field) { Queue::fake(); $writeOnlyToken = $this->user->createToken('server-subsystems-write', ['write'])->plainTextToken; diff --git a/tests/Feature/ServerMetadataTest.php b/tests/Feature/ServerMetadataTest.php index 5d521a71e6..520f28f7ed 100644 --- a/tests/Feature/ServerMetadataTest.php +++ b/tests/Feature/ServerMetadataTest.php @@ -203,9 +203,9 @@ it('calls gatherServerMetadata during ValidateAndInstall when docker version is // Call the action on the mounted instance: a Livewire request round trip // would rehydrate the server from the database and drop the mock. - Livewire::test(ValidateAndInstall::class, ['server' => $serverMock]) - ->instance() - ->validateDockerVersion(); + $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance(); + $component->uptime = '1 day'; + $component->validateDockerVersion(); }); it('does not call gatherServerMetadata when docker version validation fails', function () { @@ -216,9 +216,9 @@ it('does not call gatherServerMetadata when docker version validation fails', fu // Call the action on the mounted instance: a Livewire request round trip // would rehydrate the server from the database and drop the mock. - Livewire::test(ValidateAndInstall::class, ['server' => $serverMock]) - ->instance() - ->validateDockerVersion(); + $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance(); + $component->uptime = '1 day'; + $component->validateDockerVersion(); }); it('stores detected NVIDIA GPUs and the container toolkit when gathering server metadata', function (array $gpuLines, array $toolkitLines, array $expectedGpus, bool $expectedToolkit) { diff --git a/tests/Feature/ServerValidationCooldownTest.php b/tests/Feature/ServerValidationCooldownTest.php new file mode 100644 index 0000000000..c88fa3ceb5 --- /dev/null +++ b/tests/Feature/ServerValidationCooldownTest.php @@ -0,0 +1,190 @@ + 0, 'is_api_enabled' => true]); + config(['constants.coolify.self_hosted' => false, 'constants.ssh.mux_enabled' => false]); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user->id, ['role' => 'owner']); + session(['currentTeam' => $this->team]); + $this->server = Server::factory()->create([ + 'team_id' => $this->team->id, + 'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id, + ]); + $this->token = $this->user->createToken('server-validation', ['write'])->plainTextToken; + + Queue::fake(); +}); + +function validateServerViaApi(Server $server): TestResponse +{ + return test()->withHeaders(['Authorization' => 'Bearer '.test()->token]) + ->postJson("/api/v1/servers/{$server->uuid}/validate"); +} + +it('blocks API validations of one IP after 10 attempts on cloud', function () { + foreach (range(1, 10) as $attempt) { + validateServerViaApi($this->server)->assertCreated(); + } + + validateServerViaApi($this->server) + ->assertStatus(429) + ->assertHeader('Retry-After') + ->assertJsonPath('message', fn (string $message): bool => str_starts_with($message, 'Too many server validations.')); + ValidateServer::assertPushed(10); +}); + +it('uses the rate limit window from the instance settings', function () { + InstanceSettings::findOrFail(0)->forceFill(['server_validation_rate_limit_seconds' => 120])->save(); + Once::flush(); + + foreach (range(1, 10) as $attempt) { + validateServerViaApi($this->server)->assertCreated(); + } + + $retryAfter = (int) validateServerViaApi($this->server)->assertStatus(429)->headers->get('Retry-After'); + expect($retryAfter)->toBeGreaterThan(0)->toBeLessThanOrEqual(120); +}); + +it('uses the maximum failed validations per IP and per team from the instance settings', function () { + InstanceSettings::findOrFail(0)->forceFill([ + 'server_validation_max_failures_per_ip' => 2, + 'server_validation_max_failures_per_team' => 3, + ])->save(); + Once::flush(); + $otherServer = Server::factory()->create(['team_id' => $this->team->id]); + + validateServerViaApi($this->server)->assertCreated(); + validateServerViaApi($this->server)->assertCreated(); + validateServerViaApi($this->server)->assertStatus(429); + + validateServerViaApi($otherServer)->assertCreated(); + validateServerViaApi($otherServer)->assertStatus(429); +}); + +it('lets an instance admin change the validation limits on cloud', function () { + $rootTeam = Team::factory()->create(['id' => 0]); + $rootTeam->members()->attach($this->user->id, ['role' => 'owner']); + $this->actingAs($this->user); + session(['currentTeam' => $rootTeam]); + + Livewire::test(Advanced::class) + ->assertSet('server_validation_rate_limit_seconds', 600) + ->set('server_validation_rate_limit_seconds', 30) + ->call('submit') + ->assertDispatched('error'); + + expect(InstanceSettings::findOrFail(0)->server_validation_rate_limit_seconds)->toBe(600); + + Livewire::test(Advanced::class) + ->assertSet('server_validation_max_failures_per_ip', 10) + ->assertSet('server_validation_max_failures_per_team', 30) + ->set('server_validation_rate_limit_seconds', 1800) + ->set('server_validation_max_failures_per_ip', 5) + ->set('server_validation_max_failures_per_team', 50) + ->call('submit') + ->assertHasNoErrors(); + + expect(InstanceSettings::findOrFail(0)) + ->server_validation_rate_limit_seconds->toBe(1800) + ->server_validation_max_failures_per_ip->toBe(5) + ->server_validation_max_failures_per_team->toBe(50); +}); + +it('blocks API validations of a team after 30 attempts on cloud, also for different IPs', function () { + $servers = Server::factory()->count(31)->create(['team_id' => $this->team->id]); + + foreach ($servers->take(30) as $server) { + validateServerViaApi($server)->assertCreated(); + } + + validateServerViaApi($servers->last())->assertStatus(429); +}); + +it('blocks validations of one IP from different teams', function () { + $otherTeam = Team::factory()->create(); + + foreach (range(1, 10) as $attempt) { + Server::throttleValidation($this->team->id, $this->server->ip); + } + + expect(fn () => Server::throttleValidation($otherTeam->id, $this->server->ip)) + ->toThrow(ThrottleRequestsException::class); +}); + +it('does not limit validations on self-hosted instances', function () { + config(['constants.coolify.self_hosted' => true]); + + foreach (range(1, 31) as $attempt) { + validateServerViaApi($this->server)->assertCreated(); + } +}); + +it('gives the attempt back when the connection succeeds', function () { + Process::fake(); + + foreach (range(1, 12) as $attempt) { + Server::throttleValidation($this->team->id, $this->server->ip); + expect($this->server->validateConnection()['uptime'])->toBeTrue(); + } + + validateServerViaApi($this->server)->assertCreated(); +}); + +it('tries a failed SSH login only once', function () { + config(['constants.ssh.retry_base_delay' => 0, 'constants.ssh.retry_max_delay' => 0]); + Process::fake(['*' => Process::result(errorOutput: 'ssh: connect to host 192.0.2.1 port 22: Connection refused', exitCode: 255)]); + + expect($this->server->validateConnection()['uptime'])->toBeFalse(); + + Process::assertRanTimes(fn ($process): bool => str_contains(implode(' ', (array) $process->command), 'ssh'), 1); +}); + +it('does not connect to the server when the retry button is throttled on cloud', function () { + foreach (range(1, 10) as $attempt) { + Server::throttleValidation($this->team->id, $this->server->ip); + } + + $this->actingAs($this->user); + $serverMock = Mockery::mock($this->server)->makePartial(); + $serverMock->shouldNotReceive('validateConnection'); + + $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance(); + $component->retry(); + + expect($component->uptime)->toBeNull(); +}); + +it('skips the later validation steps when no connection succeeded', function () { + $this->actingAs($this->user); + $serverMock = Mockery::mock($this->server)->makePartial(); + $serverMock->shouldNotReceive('validateOS'); + $serverMock->shouldNotReceive('validatePrerequisites'); + $serverMock->shouldNotReceive('validateDockerEngine'); + $serverMock->shouldNotReceive('validateDockerEngineVersion'); + + $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance(); + $component->validateOS(); + $component->validatePrerequisites(); + $component->validateDockerEngine(); + $component->validateDockerVersion(); +});