create(); $team->emailNotificationSettings()->update([ 'use_instance_email_settings' => true, 'server_unreachable_email_notifications' => true, ]); $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); $server = Server::factory()->create(['team_id' => $team->id, 'private_key_id' => $privateKey->id, 'ip' => '10.0.0.1']); $server->settings->forceFill(['ssh_host_key' => $pinnedKey, 'is_reachable' => true, 'is_usable' => true])->save(); return $server; } /** * Fake ssh: the host key scan writes $scanKey to its known_hosts file; every other command returns the probe result. */ function fakeHardeningSsh(?string $scanKey, string $probeError = '', int $probeExit = 0): void { Process::fake(function (PendingProcess $process) use ($scanKey, $probeError, $probeExit) { if (str_contains($process->command, 'PubkeyAuthentication=no')) { if ($scanKey !== null && preg_match("/UserKnownHostsFile='([^']+)'/", $process->command, $matches)) { file_put_contents($matches[1], "alias {$scanKey}\n"); } return Process::result(errorOutput: 'Permission denied (publickey).', exitCode: 255); } return Process::result(output: '{"Server":{"Version":"27.0.0"}}', errorOutput: $probeError, exitCode: $probeExit); }); } function hardeningScanCount(): int { $count = 0; Process::assertRan(function (PendingProcess $process) use (&$count) { $count += str_contains($process->command, 'PubkeyAuthentication=no') ? 1 : 0; return true; }); return $count; } /** * The stderr of ssh when the server sends another key than the pinned one, as OpenSSH prints it. */ function hardeningMismatchError(Server $server, string $sentKey, bool $strict): string { $fingerprint = explode(' ', ServerSetting::sshHostKeyFingerprints($sentKey)[0])[0]; return "@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @\n" ."The fingerprint for the ED25519 key sent by the remote host is\n{$fingerprint}.\n" ." ssh-keygen -f '/var/www/html/storage/app/ssh/known_hosts/known_hosts_{$server->uuid}_0' -R 'coolify-{$server->uuid}'\n" .($strict ? "Host key for coolify-{$server->uuid} has changed and you have requested strict checking.\nHost key verification failed.\n" : ''); } beforeEach(function () { config(['constants.ssh.mux_enabled' => false, 'constants.ssh.host_key_checking' => 'warn']); Storage::fake('ssh-keys'); Cache::flush(); Sleep::fake(); Server::flushIdentityMap(); }); it('does not scan the host key while it builds a command for a server that is unreachable', function () { fakeHardeningSsh(HARDENING_KEY_B); $server = makeHardeningServer(null); $server->forceFill(['unreachable_count' => 1])->save(); SshMultiplexingHelper::generateSshCommand($server, 'ls'); Process::assertNothingRan(); }); it('does not scan again for ten minutes after a failed scan', function () { fakeHardeningSsh(null); $server = makeHardeningServer(null); SshMultiplexingHelper::generateSshCommand($server, 'ls'); $this->travel(9)->minutes(); SshMultiplexingHelper::generateSshCommand($server, 'ls'); expect(hardeningScanCount())->toBe(1); }); it('learns the host key in the connection check after a successful connection', function () { fakeHardeningSsh(HARDENING_KEY_B); $server = makeHardeningServer(null); $server->forceFill(['unreachable_count' => 1])->save(); (new ServerConnectionCheckJob($server->fresh()))->handle(); expect($server->settings->fresh()->ssh_host_key)->toBe(HARDENING_KEY_B); }); it('learns the host key in the connection check of an unreachable server in enforce mode', function () { config(['constants.ssh.host_key_checking' => 'enforce']); fakeHardeningSsh(HARDENING_KEY_B); $server = makeHardeningServer(null); $server->forceFill(['unreachable_count' => 1])->save(); (new ServerConnectionCheckJob($server->fresh()))->handle(); expect($server->settings->fresh()) ->ssh_host_key->toBe(HARDENING_KEY_B) ->is_reachable->toBeTrue(); }); it('does not scan again in the connection check for ten minutes after a failed scan', function (string $mode) { config(['constants.ssh.host_key_checking' => $mode]); fakeHardeningSsh(HARDENING_KEY_B); $server = makeHardeningServer(null); $server->forceFill(['unreachable_count' => 1])->save(); Cache::put('ssh_host_key_scan_failed_'.$server->uuid, 'Connection refused', 600); (new ServerConnectionCheckJob($server->fresh()))->handle(); Process::assertDidntRun(fn (PendingProcess $process) => str_contains($process->command, 'PubkeyAuthentication=no')); expect($server->settings->fresh()->ssh_host_key)->toBeNull(); $this->travel(11)->minutes(); (new ServerConnectionCheckJob($server->fresh()))->handle(); expect(hardeningScanCount())->toBe(1) ->and($server->settings->fresh()) ->ssh_host_key->toBe(HARDENING_KEY_B) ->is_reachable->toBeTrue(); })->with(['warn', 'enforce']); it('reports a normal connection error when the host key check fails but the server sends the pinned key', function () { config(['constants.ssh.host_key_checking' => 'enforce']); fakeHardeningSsh(HARDENING_PINNED_KEY, "No ED25519 host key is known for coolify-x and you have requested strict checking.\nHost key verification failed.\n", 255); $server = makeHardeningServer(); (new ServerConnectionCheckJob($server->fresh()))->handle(); $server->refresh(); expect($server->unreachable_count)->toBe(1) ->and($server->settings->fresh()) ->is_usable->toBeTrue() ->ssh_host_key_pending->toBeNull(); }); it('notifies once when a load balancer alternates between host keys', function () { Notification::fake(); $server = makeHardeningServer(); foreach ([HARDENING_KEY_B, HARDENING_KEY_C, HARDENING_KEY_B, HARDENING_KEY_C] as $key) { fakeHardeningSsh($key); expect($server->recordHostKeyMismatch())->toBeTrue(); $this->travel(2)->hours(); } Notification::assertSentToTimes($server->team, HostKeyChanged::class, 2); expect($server->settings->fresh()->ssh_host_key_pending)->toBe(HARDENING_KEY_C); }); it('does not scan the host key again when ssh shows the fingerprint of the pending key', function () { $server = makeHardeningServer(); $server->settings->forceFill(['ssh_host_key_pending' => HARDENING_KEY_B])->save(); fakeHardeningSsh(HARDENING_KEY_B, hardeningMismatchError($server, HARDENING_KEY_B, strict: false)); (new ServerConnectionCheckJob($server->fresh()))->handle(); expect(hardeningScanCount())->toBe(0); }); it('scans the host key at most every ten minutes while a changed key is pending', function () { $server = makeHardeningServer(); $server->settings->forceFill(['ssh_host_key_pending' => HARDENING_KEY_B])->save(); fakeHardeningSsh(HARDENING_KEY_B); expect($server->recordHostKeyMismatch("Host key verification failed.\n"))->toBeTrue(); $this->travel(5)->minutes(); expect($server->recordHostKeyMismatch("Host key verification failed.\n"))->toBeTrue(); expect(hardeningScanCount())->toBe(1); $this->travel(6)->minutes(); $server->recordHostKeyMismatch("Host key verification failed.\n"); expect(hardeningScanCount())->toBe(2); }); it('forgets the pinned host key when a placeholder IP gets the real address', function () { $server = makeHardeningServer(); Server::query()->whereKey($server->id)->update(['ip' => '1.2.3.4']); $server->refresh(); expect($server->backfillPlaceholderIp('203.0.113.5'))->toBeTrue() ->and($server->settings->fresh()->ssh_host_key)->toBeNull(); }); it('quotes the host key alias in the ssh command', function () { fakeHardeningSsh(null); $server = makeHardeningServer(); expect(SshMultiplexingHelper::generateSshCommand($server, 'ls'))->toContain("-o HostKeyAlias='coolify-{$server->uuid}' "); }); it('deletes the known_hosts file of an old host key', function () { fakeHardeningSsh(null); $server = makeHardeningServer(); preg_match("/UserKnownHostsFile='([^']+)'/", SshMultiplexingHelper::generateSshCommand($server, 'ls'), $old); $server->settings->forceFill(['ssh_host_key' => HARDENING_KEY_B])->save(); preg_match("/UserKnownHostsFile='([^']+)'/", SshMultiplexingHelper::generateSshCommand($server->fresh(), 'ls'), $new); expect(file_exists($old[1]))->toBeFalse() ->and(file_exists($new[1]))->toBeTrue(); $server->forceDelete(); expect(file_exists($new[1]))->toBeFalse(); }); it('does not check the connection when the ssh of another program reports a host key failure', function () { Queue::fake([ServerConnectionCheckJob::class]); fakeHardeningSsh(null, "Host key verification failed.\nfatal: Could not read from remote repository.\n", 128); $server = makeHardeningServer(); expect(fn () => instant_remote_process(['git clone git@github.com:a/b'], $server))->toThrow(RuntimeException::class); Queue::assertNotPushed(ServerConnectionCheckJob::class); });