diff --git a/hosts/atlas/default.nix b/hosts/atlas/default.nix
index 9d9eb84a..691edb9e 100644
--- a/hosts/atlas/default.nix
+++ b/hosts/atlas/default.nix
@@ -9,7 +9,6 @@
     ../common/global
     ../common/optional/ckb-next.nix
     ../common/optional/misterio-greetd.nix
-    ../common/optional/passwordless-sudo.nix
     ../common/optional/pipewire.nix
     ../common/optional/podman.nix
     ../common/optional/postgres.nix
diff --git a/hosts/common/global/openssh.nix b/hosts/common/global/openssh.nix
index e12eba0c..fe03ab0a 100644
--- a/hosts/common/global/openssh.nix
+++ b/hosts/common/global/openssh.nix
@@ -25,4 +25,7 @@ in
       type = "ed25519";
     }
   ];
+
+  # Passwordless sudo when SSH'ing with keys
+  security.pam.enableSSHAgentAuth = true;
 }
diff --git a/hosts/common/optional/passwordless-sudo.nix b/hosts/common/optional/passwordless-sudo.nix
deleted file mode 100644
index f3c1bc05..00000000
--- a/hosts/common/optional/passwordless-sudo.nix
+++ /dev/null
@@ -1,3 +0,0 @@
-{
-  security.sudo.extraConfig = "%wheel ALL = (ALL) NOPASSWD: ALL";
-}
diff --git a/hosts/merope/default.nix b/hosts/merope/default.nix
index e34cbd8a..f84d885b 100644
--- a/hosts/merope/default.nix
+++ b/hosts/merope/default.nix
@@ -6,7 +6,6 @@
     ./hardware-configuration.nix
     ../common/global
     ../common/optional/acme.nix
-    ../common/optional/passwordless-sudo.nix
     ../common/optional/podman.nix
     ../common/optional/postgres.nix
     ../common/optional/tailscale.nix
diff --git a/hosts/pleione/default.nix b/hosts/pleione/default.nix
index be5f95dc..f6d9527b 100644
--- a/hosts/pleione/default.nix
+++ b/hosts/pleione/default.nix
@@ -9,7 +9,6 @@
     ../common/global
     ../common/optional/misterio-greetd.nix
     ../common/optional/networkmanager.nix
-    ../common/optional/passwordless-sudo.nix
     ../common/optional/pipewire.nix
     ../common/optional/podman.nix
     ../common/optional/postgres.nix