fix(server): rate limit failed server validations on cloud

app/Helpers/SshRetryHandler.php

@@ -27,8 +27,8 @@ class SshRetryHandler
     /**
      * Convenience static method for retry execution
      */
-    public static function retry(callable $callback, array $context = [], bool $throwError = true)
+    public static function retry(callable $callback, array $context = [], bool $throwError = true, ?int $maxAttempts = null)
     {
-        return self::instance()->executeWithSshRetry($callback, $context, $throwError);
+        return self::instance()->executeWithSshRetry($callback, $context, $throwError, $maxAttempts);
     }
 }

app/Http/Controllers/Api/DestinationsController.php

@@ -284,6 +284,10 @@ class DestinationsController extends Controller
 
     private function existingServerNetworkResponse(Server $server, string $network): ?JsonResponse
     {
+        if (! $server->isFunctional()) {
+            return response()->json(['message' => 'Server is not functional.'], 400);
+        }
+
         try {
             $output = instant_remote_process(["docker network ls --format '{{.Name}}'"], $server);
         } catch (\Throwable) {

app/Http/Controllers/Api/ServerLogDrainsController.php

@@ -274,10 +274,12 @@ class ServerLogDrainsController extends Controller
         $server->refresh();
 
         // Match Livewire instantSave: start or stop the drain service after settings change.
-        if ($server->isLogDrainEnabled()) {
-            StartLogDrain::dispatch($server);
-        } else {
-            StopLogDrain::dispatch($server);
+        if ($server->isFunctional()) {
+            if ($server->isLogDrainEnabled()) {
+                StartLogDrain::dispatch($server);
+            } else {
+                StopLogDrain::dispatch($server);
+            }
         }
 
         auditLog('api.server.log_drains.updated', [

app/Http/Controllers/Api/ServerProxyController.php

@@ -389,6 +389,10 @@ class ServerProxyController extends Controller
             ], 422);
         }
 
+        if (! $server->isFunctional()) {
+            return response()->json(['message' => 'Server is not functional.'], 400);
+        }
+
         SaveProxyConfiguration::run($server, $configuration);
 
         auditLog('api.server.proxy.configuration_saved', [
@@ -443,6 +447,10 @@ class ServerProxyController extends Controller
 
         $this->authorize('manageProxy', $server);
 
+        if (! $server->isFunctional()) {
+            return response()->json(['message' => 'Server is not functional.'], 400);
+        }
+
         RestartProxyJob::dispatch($server);
 
         auditLog('api.server.proxy.restarted', [

app/Http/Controllers/Api/ServersController.php

@@ -603,6 +603,9 @@ class ServersController extends Controller
         if ($foundServer) {
             return response()->json(['message' => 'A server with this IP/Domain is already in use.'], 400);
         }
+        if ($request->instant_validate) {
+            ModelsServer::throttleValidation($teamId, $request->ip);
+        }
 
         $proxyType = $request->proxy_type ? str($request->proxy_type)->upper() : ProxyTypes::TRAEFIK->value;
 
@@ -772,6 +775,9 @@ class ServersController extends Controller
             return response()->json(['message' => 'Server not found.'], 404);
         }
         $this->authorize('update', $server);
+        if ($request->instant_validate) {
+            ModelsServer::throttleValidation($teamId, $request->ip ?? $server->ip);
+        }
         if ($request->proxy_type) {
             $validProxyTypes = collect(ProxyTypes::cases())->map(function ($proxyType) {
                 return str($proxyType->value)->lower();
@@ -1137,6 +1143,7 @@ class ServersController extends Controller
         if ($install && ($response = $this->deployAbilityForbiddenResponse($request, 'Installing the server runs commands on it and starts the proxy, so it needs a token with the deploy permission.'))) {
             return $response;
         }
+        ModelsServer::throttleValidation($teamId, $server->ip);
         if ($install) {
             ValidateAndInstallServerJob::dispatch($server);
         } else {

app/Livewire/Server/PrivateKey/Show.php

@@ -5,6 +5,7 @@ namespace App\Livewire\Server\PrivateKey;
 use App\Models\PrivateKey;
 use App\Models\Server;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Illuminate\Http\Exceptions\ThrottleRequestsException;
 use Illuminate\Support\Facades\DB;
 use Livewire\Component;
 
@@ -44,8 +45,13 @@ class Show extends Component
 
             return;
         }
+        $this->authorize('update', $this->server);
+        try {
+            Server::throttleValidation($this->server->team_id, $this->server->ip);
+        } catch (ThrottleRequestsException $e) {
+            return handleError($e, $this);
+        }
         try {
-            $this->authorize('update', $this->server);
             DB::transaction(function () use ($ownedPrivateKey) {
                 $this->server->privateKey()->associate($ownedPrivateKey);
                 $this->server->save();
@@ -95,6 +101,7 @@ class Show extends Component
     public function checkConnection()
     {
         try {
+            Server::throttleValidation($this->server->team_id, $this->server->ip);
             ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection();
             if ($uptime) {
                 $this->dispatch('success', 'Server is reachable.');

app/Livewire/Server/Show.php

@@ -442,6 +442,7 @@ class Show extends Component
         try {
             $this->authorize('update', $this->server);
             $this->syncData(true);
+            Server::throttleValidation($this->server->team_id, $this->server->ip);
             ['uptime' => $uptime, 'error' => $error] = $this->server->validateConnection();
             if ($uptime) {
                 $this->dispatch('success', 'Server is reachable.');

app/Livewire/Server/ValidateAndInstall.php

@@ -8,6 +8,7 @@ use App\Events\ServerValidated;
 use App\Jobs\CheckAndStartSentinelJob;
 use App\Models\Server;
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
+use Livewire\Attributes\Locked;
 use Livewire\Component;
 
 class ValidateAndInstall extends Component
@@ -22,6 +23,7 @@ class ValidateAndInstall extends Component
 
     public bool $install = true;
 
+    #[Locked]
     public $uptime = null;
 
     public $supported_os_type = null;
@@ -113,6 +115,7 @@ class ValidateAndInstall extends Component
     {
         try {
             $this->authorize('update', $this->server);
+            Server::throttleValidation($this->server->team_id, $this->server->ip);
             if ($this->server->vultr_instance_id) {
                 $status = $this->server->refreshVultrState();
                 $this->server->refresh();
@@ -177,6 +180,10 @@ class ValidateAndInstall extends Component
     {
         $this->authorize('update', $this->server);
 
+        if (! $this->uptime) {
+            return;
+        }
+
         $this->supported_os_type = $this->server->validateOS();
         if (! $this->supported_os_type) {
             $this->error = 'Server OS type is not supported. Please install Docker manually before continuing: <a target="_blank" class="underline" href="https://docs.docker.com/engine/install/#server">documentation</a>.';
@@ -193,6 +200,10 @@ class ValidateAndInstall extends Component
     {
         $this->authorize('update', $this->server);
 
+        if (! $this->uptime) {
+            return;
+        }
+
         $validationResult = $this->server->validatePrerequisites();
         $this->prerequisites_installed = $validationResult['success'];
         if (! $validationResult['success']) {
@@ -233,6 +244,10 @@ class ValidateAndInstall extends Component
     {
         $this->authorize('update', $this->server);
 
+        if (! $this->uptime) {
+            return;
+        }
+
         $this->docker_installed = $this->server->validateDockerEngine();
         $this->docker_compose_installed = $this->server->validateDockerCompose();
         if (! $this->docker_installed || ! $this->docker_compose_installed) {
@@ -271,6 +286,10 @@ class ValidateAndInstall extends Component
     {
         $this->authorize('update', $this->server);
 
+        if (! $this->uptime) {
+            return;
+        }
+
         if ($this->server->isSwarm()) {
             $swarmInstalled = $this->server->validateDockerSwarm();
             if ($swarmInstalled) {

app/Livewire/Settings/Advanced.php

@@ -66,6 +66,12 @@ class Advanced extends Component
 
     public string $file_download_storage = 'stream';
 
+    public int $server_validation_rate_limit_seconds = 600;
+
+    public int $server_validation_max_failures_per_ip = 10;
+
+    public int $server_validation_max_failures_per_team = 30;
+
     public function rules()
     {
         return [
@@ -85,6 +91,9 @@ class Advanced extends Component
             'webhook_allow_localhost' => 'boolean',
             'domain_connect_private_key' => 'nullable|string',
             'image_cdn_url' => 'nullable|url|max:255',
+            'server_validation_rate_limit_seconds' => 'required|integer|min:60|max:86400',
+            'server_validation_max_failures_per_ip' => 'required|integer|min:1|max:1000',
+            'server_validation_max_failures_per_team' => 'required|integer|min:1|max:10000',
         ];
     }
 
@@ -114,6 +123,9 @@ class Advanced extends Component
             ? 's3:'.$this->settings->avatar_s3_storage_id
             : 'local';
         $this->image_cdn_url = $this->settings->image_cdn_url;
+        $this->server_validation_rate_limit_seconds = $this->settings->server_validation_rate_limit_seconds;
+        $this->server_validation_max_failures_per_ip = $this->settings->server_validation_max_failures_per_ip;
+        $this->server_validation_max_failures_per_team = $this->settings->server_validation_max_failures_per_team;
         $this->avatar_storage_options = [
             ['value' => 'local', 'label' => 'Local storage'],
             ...S3Storage::query()
@@ -200,6 +212,12 @@ class Advanced extends Component
                 return;
             }
 
+            if (isCloud()) {
+                $this->settings->server_validation_rate_limit_seconds = $this->server_validation_rate_limit_seconds;
+                $this->settings->server_validation_max_failures_per_ip = $this->server_validation_max_failures_per_ip;
+                $this->settings->server_validation_max_failures_per_team = $this->server_validation_max_failures_per_team;
+            }
+
             if (isCloud() && filled($this->domain_connect_private_key)) {
                 $this->settings->domain_connect_private_key = $this->normalizeDomainConnectPrivateKey($this->domain_connect_private_key);
                 $this->domain_connect_private_key = null;

app/Models/InstanceSettings.php

@@ -105,6 +105,9 @@ class InstanceSettings extends Model
         'webhook_allow_localhost' => 'boolean',
         'is_dashboard_force_https_enabled' => 'boolean',
         'is_passkey_enabled' => 'boolean',
+        'server_validation_rate_limit_seconds' => 'integer',
+        'server_validation_max_failures_per_ip' => 'integer',
+        'server_validation_max_failures_per_team' => 'integer',
     ];
 
     protected static function booted(): void

app/Models/Server.php

@@ -34,11 +34,13 @@ use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Relations\MorphMany;
 use Illuminate\Database\Eloquent\SoftDeletes;
+use Illuminate\Http\Exceptions\ThrottleRequestsException;
 use Illuminate\Support\Carbon;
 use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\RateLimiter;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
 use Illuminate\Support\Stringable;
@@ -1407,6 +1409,64 @@ $siteAddress {
         return ! $this->isTransferredAway();
     }
 
+    /**
+     * On Coolify Cloud, limit how often SSH validations can run against one IP and from one team.
+     * The window and the maximums are server_validation_* instance settings.
+     * Without a limit, users can make Cloud try SSH logins on hosts that they do not own.
+     * A successful connection gives its attempt back, so only failed logins use up the limit.
+     *
+     * @throws ThrottleRequestsException
+     */
+    public static function throttleValidation(int $teamId, string $ip): void
+    {
+        if (! isCloud()) {
+            return;
+        }
+
+        $limits = self::validationRateLimits($teamId, $ip);
+        foreach ($limits as $key => $maxAttempts) {
+            if (RateLimiter::tooManyAttempts($key, $maxAttempts)) {
+                $seconds = RateLimiter::availableIn($key);
+
+                throw new ThrottleRequestsException("Too many server validations. Please try again in {$seconds} seconds.", headers: ['Retry-After' => $seconds]);
+            }
+        }
+        foreach (array_keys($limits) as $key) {
+            RateLimiter::hit($key, self::validationRateLimitSeconds());
+        }
+    }
+
+    private static function validationRateLimitSeconds(): int
+    {
+        return instanceSettings()->server_validation_rate_limit_seconds;
+    }
+
+    /**
+     * @return array<string, int> rate limiter key => max attempts
+     */
+    private static function validationRateLimits(int $teamId, string $ip): array
+    {
+        $settings = instanceSettings();
+
+        return [
+            'server-validation:ip:'.strtolower(self::normalizeIp($ip)) => $settings->server_validation_max_failures_per_ip,
+            'server-validation:team:'.$teamId => $settings->server_validation_max_failures_per_team,
+        ];
+    }
+
+    private function refundValidationAttempt(): void
+    {
+        if (! isCloud()) {
+            return;
+        }
+
+        foreach (array_keys(self::validationRateLimits($this->team_id, $this->ip)) as $key) {
+            if (RateLimiter::attempts($key) > 0) {
+                RateLimiter::decrement($key, self::validationRateLimitSeconds());
+            }
+        }
+    }
+
     public function forceEnableServer()
     {
         if ($this->isTransferredAway()) {
@@ -2333,7 +2393,9 @@ $siteAddress {
             return ['uptime' => false, 'error' => 'Server skipped.'];
         }
         try {
-            instant_remote_process(['ls /'], $this, disableMultiplexing: true);
+            // One attempt only: retries would repeat failed logins on a host that may not belong to the user.
+            instant_remote_process(['ls /'], $this, disableMultiplexing: true, maxAttempts: 1);
+            $this->refundValidationAttempt();
             if ($this->settings->is_reachable === false) {
                 $this->settings->is_reachable = true;
                 $this->settings->save();

app/Traits/SshRetryable.php

@@ -90,11 +90,12 @@ trait SshRetryable
      * @param  callable  $callback  The operation to execute
      * @param  array  $context  Context for logging (server, command, etc.)
      * @param  bool  $throwError  Whether to throw error on final failure
+     * @param  int|null  $maxAttempts  Attempts in total, defaults to constants.ssh.max_retries
      * @return mixed The result from the callback
      */
-    protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true)
+    protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true, ?int $maxAttempts = null)
     {
-        $maxRetries = config('constants.ssh.max_retries');
+        $maxRetries = $maxAttempts ?? config('constants.ssh.max_retries');
         $lastError = null;
         $lastErrorMessage = '';
         // Randomly fail the command with a key exchange error for testing

bootstrap/helpers/remoteProcess.php

@@ -216,7 +216,7 @@ function instant_remote_process_with_timeout(Collection|array $command, Server $
     );
 }
 
-function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false): ?string
+function instant_remote_process(Collection|array $command, Server $server, bool $throwError = true, bool $no_sudo = false, ?int $timeout = null, bool $disableMultiplexing = false, ?int $maxAttempts = null): ?string
 {
     $command = $command instanceof Collection ? $command->toArray() : $command;
 
@@ -248,7 +248,8 @@ function instant_remote_process(Collection|array $command, Server $server, bool
             'command_preview' => substr($command_string, 0, 100),
             'function' => 'instant_remote_process',
         ],
-        $throwError
+        $throwError,
+        $maxAttempts
     );
 }
 

database/migrations/2026_10_23_090005_add_server_validation_limits_to_instance_settings_table.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        Schema::table('instance_settings', function (Blueprint $table) {
+            $table->unsignedInteger('server_validation_rate_limit_seconds')->default(600);
+            $table->unsignedInteger('server_validation_max_failures_per_ip')->default(10);
+            $table->unsignedInteger('server_validation_max_failures_per_team')->default(30);
+        });
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        Schema::table('instance_settings', function (Blueprint $table) {
+            $table->dropColumn([
+                'server_validation_rate_limit_seconds',
+                'server_validation_max_failures_per_ip',
+                'server_validation_max_failures_per_team',
+            ]);
+        });
+    }
+};

resources/views/livewire/settings/advanced.blade.php

@@ -115,6 +115,20 @@
                             @endif
                         </div>
                     </x-application.settings-subsection>
+                    <x-application.settings-subsection id="server-validation-section" title="Server validation"
+                        helper="Limits failed SSH validations, so users cannot make Coolify Cloud try logins on hosts that they do not own. Successful connections do not count.">
+                        <div class="grid gap-4 lg:grid-cols-3">
+                            <x-forms.input id="server_validation_rate_limit_seconds" type="number" min="60" max="86400"
+                                label="Rate limit window (seconds)"
+                                helper="The period in which Coolify counts failed validations. From 60 to 86400 seconds." />
+                            <x-forms.input id="server_validation_max_failures_per_ip" type="number" min="1" max="1000"
+                                label="Max failed validations per IP"
+                                helper="For one target IP, from all teams together, in the window. From 1 to 1000." />
+                            <x-forms.input id="server_validation_max_failures_per_team" type="number" min="1" max="10000"
+                                label="Max failed validations per team"
+                                helper="For one team, to all IPs together, in the window. From 1 to 10000." />
+                        </div>
+                    </x-application.settings-subsection>
                 @endif
                 <x-application.settings-subsection id="endpoint-section" title="Outbound endpoints">
                     <div class="flex flex-col gap-4">

tests/Feature/Api/DestinationsApiTest.php

@@ -44,6 +44,7 @@ beforeEach(function () {
         'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,
     ]);
     $this->destination = StandaloneDocker::where('server_id', $this->server->id)->first();
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
 });
 
 function destinationsApiHeaders(string $bearerToken): array
@@ -130,6 +131,18 @@ describe('POST /api/v1/servers/{server_uuid}/destinations', function () {
             ->exists())->toBeTrue();
     });
 
+    test('does not connect to a server that is not validated', function () {
+        $this->server->settings->update(['is_reachable' => false, 'is_usable' => false]);
+
+        $this->withHeaders(destinationsApiHeaders($this->bearerToken))
+            ->postJson("/api/v1/servers/{$this->server->uuid}/destinations", ['network' => 'api-standalone-network'])
+            ->assertStatus(400)
+            ->assertJsonPath('message', 'Server is not functional.');
+
+        Process::assertNothingRan();
+        expect(StandaloneDocker::where('network', 'api-standalone-network')->exists())->toBeFalse();
+    });
+
     test('requires a write token', function () {
         $readOnlyToken = destinationsApiToken($this->user, $this->team, ['read']);
 

tests/Feature/Api/ServerDeployAbilityTest.php

@@ -41,6 +41,7 @@ beforeEach(function () {
     ]);
     $this->server->proxy->set('type', ProxyTypes::TRAEFIK->value);
     $this->server->save();
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
 });
 
 function serverDeployAbilityHeaders(User $user, int $teamId, array $abilities): array

tests/Feature/Api/ServerProxyApiTest.php

@@ -216,6 +216,7 @@ test('PATCH /api/v1/servers/{uuid}/proxy can change proxy type asynchronously',
 });
 
 test('PUT /api/v1/servers/{uuid}/proxy/configuration saves base64 configuration via action', function () {
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
     $compose = "services:\n  traefik:\n    image: traefik:v3.5\n";
 
     SaveProxyConfiguration::shouldRun()
@@ -241,6 +242,7 @@ test('PUT /api/v1/servers/{uuid}/proxy/configuration rejects missing configurati
 });
 
 test('PUT /api/v1/servers/{uuid}/proxy/configuration rejects command injection without mutation', function () {
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
     $originalProxy = $this->server->proxy->toArray();
     $configuration = "services:\n  traefik:\n    ports:\n      - '`id>/tmp/pwned`:443'\n";
 
@@ -283,6 +285,7 @@ test('PUT /api/v1/servers/{uuid}/proxy/configuration requires the deploy permiss
 });
 
 test('PUT /api/v1/servers/{uuid}/proxy/configuration saves with write and deploy permissions', function () {
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
     $deployToken = serverProxyApiToken($this->user, $this->team, ['write', 'deploy']);
 
     SaveProxyConfiguration::shouldRun()->once();
@@ -318,6 +321,7 @@ test('POST /api/v1/servers/{uuid}/proxy/restart requires the deploy permission',
 });
 
 test('POST /api/v1/servers/{uuid}/proxy/restart works with the deploy permission', function () {
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
     Queue::fake();
     $deployToken = serverProxyApiToken($this->user, $this->team, ['deploy']);
 
@@ -329,6 +333,7 @@ test('POST /api/v1/servers/{uuid}/proxy/restart works with the deploy permission
 });
 
 test('POST /api/v1/servers/{uuid}/proxy/restart queues RestartProxyJob', function () {
+    $this->server->settings->update(['is_reachable' => true, 'is_usable' => true]);
     Queue::fake();
 
     $this->withHeaders(serverProxyApiHeaders($this->bearerToken))
@@ -342,6 +347,25 @@ test('POST /api/v1/servers/{uuid}/proxy/restart queues RestartProxyJob', functio
     );
 });
 
+test('proxy configuration and restart endpoints do not connect to a server that is not validated', function () {
+    Queue::fake();
+    SaveProxyConfiguration::shouldNotRun();
+
+    $this->withHeaders(serverProxyApiHeaders($this->bearerToken))
+        ->putJson("/api/v1/servers/{$this->server->uuid}/proxy/configuration", [
+            'configuration' => base64_encode("services:\n  traefik:\n    image: traefik:v3.5\n"),
+        ])
+        ->assertStatus(400)
+        ->assertJsonPath('message', 'Server is not functional.');
+
+    $this->withHeaders(serverProxyApiHeaders($this->bearerToken))
+        ->postJson("/api/v1/servers/{$this->server->uuid}/proxy/restart")
+        ->assertStatus(400)
+        ->assertJsonPath('message', 'Server is not functional.');
+
+    Queue::assertNotPushed(RestartProxyJob::class);
+});
+
 test('proxy endpoints require authentication', function () {
     $this->getJson("/api/v1/servers/{$this->server->uuid}/proxy")->assertUnauthorized();
     $this->patchJson("/api/v1/servers/{$this->server->uuid}/proxy", ['redirect_enabled' => false])->assertUnauthorized();

tests/Feature/Api/ServerSubsystemsApiTest.php

@@ -1,6 +1,8 @@
 <?php
 
+use App\Actions\Server\StartLogDrain;
 use App\Actions\Server\StartSentinel;
+use App\Actions\Server\StopLogDrain;
 use App\Jobs\DockerCleanupJob;
 use App\Models\DockerCleanupExecution;
 use App\Models\InstanceSettings;
@@ -188,6 +190,19 @@ describe('Log drains API', function () {
             ->and($settings->logdrain_axiom_api_key)->toBe('axiom-key-123');
     });
 
+    test('PATCH saves log drain settings without connecting to a server that is not validated', function () {
+        Queue::fake();
+
+        $this->withHeaders(serverSubsystemsHeaders())
+            ->patchJson("/api/v1/servers/{$this->server->uuid}/log-drains", [
+                'is_logdrain_axiom_enabled' => false,
+            ])
+            ->assertOk();
+
+        StopLogDrain::assertNotPushed();
+        StartLogDrain::assertNotPushed();
+    });
+
     test('PATCH requires the deploy permission to change the custom log drain config', function (string $field) {
         Queue::fake();
         $writeOnlyToken = $this->user->createToken('server-subsystems-write', ['write'])->plainTextToken;

tests/Feature/ServerMetadataTest.php

@@ -203,9 +203,9 @@ it('calls gatherServerMetadata during ValidateAndInstall when docker version is
 
     // Call the action on the mounted instance: a Livewire request round trip
     // would rehydrate the server from the database and drop the mock.
-    Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])
-        ->instance()
-        ->validateDockerVersion();
+    $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance();
+    $component->uptime = '1 day';
+    $component->validateDockerVersion();
 });
 
 it('does not call gatherServerMetadata when docker version validation fails', function () {
@@ -216,9 +216,9 @@ it('does not call gatherServerMetadata when docker version validation fails', fu
 
     // Call the action on the mounted instance: a Livewire request round trip
     // would rehydrate the server from the database and drop the mock.
-    Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])
-        ->instance()
-        ->validateDockerVersion();
+    $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance();
+    $component->uptime = '1 day';
+    $component->validateDockerVersion();
 });
 
 it('stores detected NVIDIA GPUs and the container toolkit when gathering server metadata', function (array $gpuLines, array $toolkitLines, array $expectedGpus, bool $expectedToolkit) {

tests/Feature/ServerValidationCooldownTest.php

@@ -0,0 +1,190 @@
+<?php
+
+use App\Actions\Server\ValidateServer;
+use App\Livewire\Server\ValidateAndInstall;
+use App\Livewire\Settings\Advanced;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use App\Models\User;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Exceptions\ThrottleRequestsException;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Queue;
+use Illuminate\Support\Once;
+use Illuminate\Testing\TestResponse;
+use Livewire\Livewire;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    InstanceSettings::forceCreate(['id' => 0, 'is_api_enabled' => true]);
+    config(['constants.coolify.self_hosted' => false, 'constants.ssh.mux_enabled' => false]);
+
+    $this->team = Team::factory()->create();
+    $this->user = User::factory()->create();
+    $this->team->members()->attach($this->user->id, ['role' => 'owner']);
+    session(['currentTeam' => $this->team]);
+    $this->server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'private_key_id' => PrivateKey::factory()->create(['team_id' => $this->team->id])->id,
+    ]);
+    $this->token = $this->user->createToken('server-validation', ['write'])->plainTextToken;
+
+    Queue::fake();
+});
+
+function validateServerViaApi(Server $server): TestResponse
+{
+    return test()->withHeaders(['Authorization' => 'Bearer '.test()->token])
+        ->postJson("/api/v1/servers/{$server->uuid}/validate");
+}
+
+it('blocks API validations of one IP after 10 attempts on cloud', function () {
+    foreach (range(1, 10) as $attempt) {
+        validateServerViaApi($this->server)->assertCreated();
+    }
+
+    validateServerViaApi($this->server)
+        ->assertStatus(429)
+        ->assertHeader('Retry-After')
+        ->assertJsonPath('message', fn (string $message): bool => str_starts_with($message, 'Too many server validations.'));
+    ValidateServer::assertPushed(10);
+});
+
+it('uses the rate limit window from the instance settings', function () {
+    InstanceSettings::findOrFail(0)->forceFill(['server_validation_rate_limit_seconds' => 120])->save();
+    Once::flush();
+
+    foreach (range(1, 10) as $attempt) {
+        validateServerViaApi($this->server)->assertCreated();
+    }
+
+    $retryAfter = (int) validateServerViaApi($this->server)->assertStatus(429)->headers->get('Retry-After');
+    expect($retryAfter)->toBeGreaterThan(0)->toBeLessThanOrEqual(120);
+});
+
+it('uses the maximum failed validations per IP and per team from the instance settings', function () {
+    InstanceSettings::findOrFail(0)->forceFill([
+        'server_validation_max_failures_per_ip' => 2,
+        'server_validation_max_failures_per_team' => 3,
+    ])->save();
+    Once::flush();
+    $otherServer = Server::factory()->create(['team_id' => $this->team->id]);
+
+    validateServerViaApi($this->server)->assertCreated();
+    validateServerViaApi($this->server)->assertCreated();
+    validateServerViaApi($this->server)->assertStatus(429);
+
+    validateServerViaApi($otherServer)->assertCreated();
+    validateServerViaApi($otherServer)->assertStatus(429);
+});
+
+it('lets an instance admin change the validation limits on cloud', function () {
+    $rootTeam = Team::factory()->create(['id' => 0]);
+    $rootTeam->members()->attach($this->user->id, ['role' => 'owner']);
+    $this->actingAs($this->user);
+    session(['currentTeam' => $rootTeam]);
+
+    Livewire::test(Advanced::class)
+        ->assertSet('server_validation_rate_limit_seconds', 600)
+        ->set('server_validation_rate_limit_seconds', 30)
+        ->call('submit')
+        ->assertDispatched('error');
+
+    expect(InstanceSettings::findOrFail(0)->server_validation_rate_limit_seconds)->toBe(600);
+
+    Livewire::test(Advanced::class)
+        ->assertSet('server_validation_max_failures_per_ip', 10)
+        ->assertSet('server_validation_max_failures_per_team', 30)
+        ->set('server_validation_rate_limit_seconds', 1800)
+        ->set('server_validation_max_failures_per_ip', 5)
+        ->set('server_validation_max_failures_per_team', 50)
+        ->call('submit')
+        ->assertHasNoErrors();
+
+    expect(InstanceSettings::findOrFail(0))
+        ->server_validation_rate_limit_seconds->toBe(1800)
+        ->server_validation_max_failures_per_ip->toBe(5)
+        ->server_validation_max_failures_per_team->toBe(50);
+});
+
+it('blocks API validations of a team after 30 attempts on cloud, also for different IPs', function () {
+    $servers = Server::factory()->count(31)->create(['team_id' => $this->team->id]);
+
+    foreach ($servers->take(30) as $server) {
+        validateServerViaApi($server)->assertCreated();
+    }
+
+    validateServerViaApi($servers->last())->assertStatus(429);
+});
+
+it('blocks validations of one IP from different teams', function () {
+    $otherTeam = Team::factory()->create();
+
+    foreach (range(1, 10) as $attempt) {
+        Server::throttleValidation($this->team->id, $this->server->ip);
+    }
+
+    expect(fn () => Server::throttleValidation($otherTeam->id, $this->server->ip))
+        ->toThrow(ThrottleRequestsException::class);
+});
+
+it('does not limit validations on self-hosted instances', function () {
+    config(['constants.coolify.self_hosted' => true]);
+
+    foreach (range(1, 31) as $attempt) {
+        validateServerViaApi($this->server)->assertCreated();
+    }
+});
+
+it('gives the attempt back when the connection succeeds', function () {
+    Process::fake();
+
+    foreach (range(1, 12) as $attempt) {
+        Server::throttleValidation($this->team->id, $this->server->ip);
+        expect($this->server->validateConnection()['uptime'])->toBeTrue();
+    }
+
+    validateServerViaApi($this->server)->assertCreated();
+});
+
+it('tries a failed SSH login only once', function () {
+    config(['constants.ssh.retry_base_delay' => 0, 'constants.ssh.retry_max_delay' => 0]);
+    Process::fake(['*' => Process::result(errorOutput: 'ssh: connect to host 192.0.2.1 port 22: Connection refused', exitCode: 255)]);
+
+    expect($this->server->validateConnection()['uptime'])->toBeFalse();
+
+    Process::assertRanTimes(fn ($process): bool => str_contains(implode(' ', (array) $process->command), 'ssh'), 1);
+});
+
+it('does not connect to the server when the retry button is throttled on cloud', function () {
+    foreach (range(1, 10) as $attempt) {
+        Server::throttleValidation($this->team->id, $this->server->ip);
+    }
+
+    $this->actingAs($this->user);
+    $serverMock = Mockery::mock($this->server)->makePartial();
+    $serverMock->shouldNotReceive('validateConnection');
+
+    $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance();
+    $component->retry();
+
+    expect($component->uptime)->toBeNull();
+});
+
+it('skips the later validation steps when no connection succeeded', function () {
+    $this->actingAs($this->user);
+    $serverMock = Mockery::mock($this->server)->makePartial();
+    $serverMock->shouldNotReceive('validateOS');
+    $serverMock->shouldNotReceive('validatePrerequisites');
+    $serverMock->shouldNotReceive('validateDockerEngine');
+    $serverMock->shouldNotReceive('validateDockerEngineVersion');
+
+    $component = Livewire::test(ValidateAndInstall::class, ['server' => $serverMock])->instance();
+    $component->validateOS();
+    $component->validatePrerequisites();
+    $component->validateDockerEngine();
+    $component->validateDockerVersion();
+});