v4.7 (#12478)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com>
Co-authored-by: Aditya Tripathi <aditya@climactic.co>

app/Models/Server.php

@@ -19,6 +19,7 @@ use App\Jobs\CheckTraefikVersionForServerJob;
 use App\Jobs\RegenerateSslCertJob;
 use App\Jobs\ServerConnectionCheckJob;
 use App\Livewire\Server\Proxy;
+use App\Notifications\Server\HostKeyChanged;
 use App\Notifications\Server\Reachable;
 use App\Notifications\Server\Unreachable;
 use App\Services\DigitalOceanService;
@@ -34,6 +35,7 @@ use Illuminate\Database\Eloquent\Builder;
 use Illuminate\Database\Eloquent\Casts\Attribute;
 use Illuminate\Database\Eloquent\Factories\HasFactory;
 use Illuminate\Database\Eloquent\Model;
+use Illuminate\Database\Eloquent\Relations\HasMany;
 use Illuminate\Database\Eloquent\Relations\MorphMany;
 use Illuminate\Database\Eloquent\SoftDeletes;
 use Illuminate\Http\Exceptions\ThrottleRequestsException;
@@ -194,6 +196,9 @@ class Server extends BaseModel
             $server->fill($payload);
         });
         static::saved(function ($server) {
+            if ($server->wasChanged(['ip', 'port'])) {
+                $server->settings?->forgetSshHostKey('address_changed');
+            }
             if ($server->wasChanged('private_key_id') || $server->privateKey?->isDirty()) {
                 refresh_server_connection($server->privateKey);
             } elseif ($server->wasChanged(['user', 'ip', 'port'])) {
@@ -271,8 +276,10 @@ class Server extends BaseModel
             }
             $server->githubRunnerConfig()->delete();
             $server->settings()->delete();
+            SshMultiplexingHelper::deleteStaleKnownHostsFiles($server->uuid, null);
             $server->sslCertificates()->delete();
             $server->notificationThrottles()->delete();
+            ImageScanPolicy::query()->where('scope_type', $server->getMorphClass())->where('scope_id', $server->id)->delete();
         });
 
         static::deleted(function (Server $server) {
@@ -443,6 +450,8 @@ class Server extends BaseModel
         $this->forceFill(['ip' => $ip]);
         $this->syncOriginalAttribute('ip');
         static::flushIdentityMap();
+        // The query update skips the saved hook, which forgets the host key of the old address.
+        $this->settings?->forgetSshHostKey('address_changed');
 
         return true;
     }
@@ -705,6 +714,16 @@ class Server extends BaseModel
         return $this->hasOne(ServerSetting::class);
     }
 
+    public function imageScans(): HasMany
+    {
+        return $this->hasMany(ImageScan::class);
+    }
+
+    public function imageSboms(): HasMany
+    {
+        return $this->hasMany(ImageSbom::class);
+    }
+
     public function dockerCleanupExecutions()
     {
         return $this->hasMany(DockerCleanupExecution::class);
@@ -2528,8 +2547,13 @@ $siteAddress {
             return ['uptime' => false, 'error' => 'Server skipped.'];
         }
         try {
+            // Enforce mode cannot connect without the key, so it reads the key first. Warn mode learns it after the connection.
+            if (SshMultiplexingHelper::hostKeyCheckingMode($this) === 'enforce') {
+                SshMultiplexingHelper::learnHostKey($this, evenIfUnreachable: true);
+            }
             // One attempt only: retries would repeat failed logins on a host that may not belong to the user.
             instant_remote_process(['ls /'], $this, disableMultiplexing: true, maxAttempts: 1);
+            SshMultiplexingHelper::learnHostKey($this, evenIfUnreachable: true);
             $this->refundValidationAttempt();
             if ($this->settings->is_reachable === false) {
                 $this->settings->is_reachable = true;
@@ -2539,16 +2563,72 @@ $siteAddress {
 
             return ['uptime' => true, 'error' => null];
         } catch (\Throwable $e) {
+            $error = $e->getMessage();
+            if ($error === SshMultiplexingHelper::HOST_KEY_CHANGED_MESSAGE && ! $this->recordHostKeyMismatch()) {
+                $error = 'SSH host key verification failed, but the server still sends the trusted host key.';
+            }
             if (! $justCheckingNewKey && $this->settings->is_reachable === true) {
                 $this->settings->is_reachable = false;
                 $this->settings->save();
                 ServerReachabilityChanged::dispatch($this);
             }
 
-            return ['uptime' => false, 'error' => $e->getMessage()];
+            return ['uptime' => false, 'error' => $error];
         }
     }
 
+    /**
+     * ssh reported a host key problem. Returns true when the server sends a host key that is not pinned, and
+     * saves that key as pending, so that an admin can compare and trust it. Returns false when the server
+     * still sends the pinned key or the key cannot be read: then it is a normal connection error.
+     * A key is reported once a day, and the team gets at most one notification an hour, because a load
+     * balancer in front of several hosts can switch between keys on every connection. While a key is
+     * pending, the server is scanned at most every ten minutes.
+     *
+     * @param  string|null  $sshErrorOutput  The stderr of ssh. When it shows the fingerprint of the pending key, no scan is needed.
+     */
+    public function recordHostKeyMismatch(?string $sshErrorOutput = null): bool
+    {
+        $settings = $this->settings;
+        if (filled($settings->ssh_host_key_pending) && preg_match_all('/SHA256:[A-Za-z0-9+\/]+/', (string) $sshErrorOutput, $matches) > 0) {
+            $pendingFingerprints = array_map(fn (string $fingerprint) => strtok($fingerprint, ' '), ServerSetting::sshHostKeyFingerprints($settings->ssh_host_key_pending));
+            if (array_diff($matches[0], $pendingFingerprints) === []) {
+                return true;
+            }
+        }
+        // Each scan is a failed login on the server (fail2ban), so a pending key is scanned again at most every ten minutes.
+        $scannedCacheKey = 'ssh_host_key_mismatch_scanned_'.$this->uuid.'_'.hash('sha256', (string) $settings->ssh_host_key);
+        if (filled($settings->ssh_host_key_pending) && Cache::has($scannedCacheKey)) {
+            return true;
+        }
+
+        $newKey = SshMultiplexingHelper::scanHostKey($this);
+        Cache::put($scannedCacheKey, true, now()->addMinutes(10));
+        if ($newKey === null || array_diff(explode("\n", $newKey), explode("\n", (string) $settings->ssh_host_key)) === []) {
+            return false;
+        }
+        if ($newKey === $settings->ssh_host_key_pending) {
+            return true;
+        }
+
+        $settings->forceFill(['ssh_host_key_pending' => $newKey])->save();
+        if (! Cache::add('ssh_host_key_reported_'.$this->uuid.'_'.hash('sha256', $newKey), true, now()->addDay())) {
+            return true;
+        }
+        Log::warning('SSH host key of server changed', ['server_uuid' => $this->uuid, 'mode' => SshMultiplexingHelper::hostKeyCheckingMode($this)]);
+        auditLog('server.ssh_host_key_changed', [
+            'team_id' => $this->team_id,
+            'server_uuid' => $this->uuid,
+            'server_name' => $this->name,
+            'fingerprints' => ServerSetting::sshHostKeyFingerprints($newKey),
+        ], 'warning');
+        if (Cache::add('ssh_host_key_changed_notified_'.$this->uuid, true, now()->addHour())) {
+            $this->team?->notify(new HostKeyChanged($this));
+        }
+
+        return true;
+    }
+
     public function installDocker()
     {
         return InstallDocker::run($this);