From e7465000b35b90ddeecb2483102cd8e0d0a6679d Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:06:44 +0200 Subject: [PATCH] fix(api): key unauthenticated rate limits by Cloudflare client IP Use auth_rate_limit_ip() rather than $request->ip() for the api, 5 and feedback rate limiters. On cloud, each client behind Cloudflare now gets its own bucket instead of sharing the proxy's bucket. Self-hosted instances ignore the CF-Connecting-IP header. Add feature tests for per-client sentinel push buckets on cloud and for ignoring the header on self-hosted. Add scripts/restore-backup-vm.sh, which restores an instance backup into an offline KVM VM to test database migrations. Bump the v4 version in other/nightly/versions.json to sha-1e9add3. --- .claude/worktrees/agent-a1f84dd00720572e0 | 1 + app/Providers/RouteServiceProvider.php | 8 +- other/nightly/versions.json | 2 +- scripts/restore-backup-vm.sh | 356 +++++++++++++++++++++ tests/Feature/ApiRateLimitClientIpTest.php | 56 ++++ 5 files changed, 418 insertions(+), 5 deletions(-) create mode 160000 .claude/worktrees/agent-a1f84dd00720572e0 create mode 100755 scripts/restore-backup-vm.sh create mode 100644 tests/Feature/ApiRateLimitClientIpTest.php diff --git a/.claude/worktrees/agent-a1f84dd00720572e0 b/.claude/worktrees/agent-a1f84dd00720572e0 new file mode 160000 index 0000000000..ee5e46f947 --- /dev/null +++ b/.claude/worktrees/agent-a1f84dd00720572e0 @@ -0,0 +1 @@ +Subproject commit ee5e46f94738cda56ad014b25ae2f82ce40eb4c6 diff --git a/app/Providers/RouteServiceProvider.php b/app/Providers/RouteServiceProvider.php index 633216ea48..24cec00b4c 100644 --- a/app/Providers/RouteServiceProvider.php +++ b/app/Providers/RouteServiceProvider.php @@ -46,17 +46,17 @@ class RouteServiceProvider extends ServiceProvider { RateLimiter::for('api', function (Request $request) { if ($request->path() === 'api/health') { - return Limit::perMinute(1000)->by($request->user()?->id ?: $request->ip()); + return Limit::perMinute(1000)->by($request->user()?->id ?: auth_rate_limit_ip($request)); } - return Limit::perMinute((int) config('api.rate_limit'))->by($request->user()?->id ?: $request->ip()); + return Limit::perMinute((int) config('api.rate_limit'))->by($request->user()?->id ?: auth_rate_limit_ip($request)); }); RateLimiter::for('5', function (Request $request) { - return Limit::perMinute(5)->by($request->user()?->id ?: $request->ip()); + return Limit::perMinute(5)->by($request->user()?->id ?: auth_rate_limit_ip($request)); }); RateLimiter::for('feedback', function (Request $request) { - return Limit::perMinute(3)->by($request->user()?->id ?: $request->ip()); + return Limit::perMinute(3)->by($request->user()?->id ?: auth_rate_limit_ip($request)); }); RateLimiter::for('login', function (Request $request) { diff --git a/other/nightly/versions.json b/other/nightly/versions.json index 7262de886a..34821f8761 100644 --- a/other/nightly/versions.json +++ b/other/nightly/versions.json @@ -1,7 +1,7 @@ { "coolify": { "v4": { - "version": "4.4" + "version": "sha-1e9add3" }, "nightly": { "version": "4.5-rc.1" diff --git a/scripts/restore-backup-vm.sh b/scripts/restore-backup-vm.sh new file mode 100755 index 0000000000..8c52c4834f --- /dev/null +++ b/scripts/restore-backup-vm.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash +# Restore a Coolify instance backup into an offline KVM VM to test database migrations. +# +# Usage: +# scripts/restore-backup-vm.sh asks for the inputs below +# scripts/restore-backup-vm.sh destroy deletes the VM, its disk, and the Coolify base images +# +# Inputs (asked interactively, so they do not go into the shell history): +# Backup file pg_dump custom-format backup (.dmp), optionally gzipped (.gz) +# APP_KEY APP_KEY of the instance that created the backup (base64:...), not shown +# Version Coolify version or image tag, e.g. 4.3.23 or sha-99c8409 (empty: latest stable) +# +# Flow: +# 1. Base image (once per version, until destroy): a separate builder VM with internet access installs +# Coolify. It never receives the backup. Its disk is kept as the base image. +# 2. The restore VM boots from a copy of the base image on a host-only network without +# internet access and without DNS. It never has internet access, so the restored +# instance cannot reach real servers, notification channels, or Stripe. +# 3. The script confirms that the VM is offline, copies the backup, replaces the database, +# sets APP_KEY, turns off Horizon and the scheduler, and starts Coolify. +# Coolify runs its migrations and seeders on start; no queued or scheduled job runs. +# +# Environment overrides: +# VM_NAME (coolify-restore-test), VM_MEMORY (4096), VM_VCPUS (4), VM_DISK_SIZE (60G), +# IMAGE_DIR (/var/lib/libvirt/images/coolify-restore-test), +# NAT_NETWORK (default), OFFLINE_NETWORK (coolify-restore-offline), +# OFFLINE_SUBNET (192.168.241) + +set -euo pipefail + +VM_NAME=${VM_NAME:-coolify-restore-test} +BUILDER_NAME=$VM_NAME-builder +VM_MEMORY=${VM_MEMORY:-4096} +VM_VCPUS=${VM_VCPUS:-4} +VM_DISK_SIZE=${VM_DISK_SIZE:-60G} +IMAGE_DIR=${IMAGE_DIR:-/var/lib/libvirt/images/coolify-restore-test} +NAT_NETWORK=${NAT_NETWORK:-default} +OFFLINE_NETWORK=${OFFLINE_NETWORK:-coolify-restore-offline} +OFFLINE_SUBNET=${OFFLINE_SUBNET:-192.168.241} +# Both VMs use the same MAC address: the guest network config matches the NIC by MAC. +VM_MAC=52:54:00:c0:1f:01 +UBUNTU_IMAGE_URL=https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img +UBUNTU_IMAGE="$IMAGE_DIR/ubuntu-noble-amd64.qcow2" +SSH_KEY="$IMAGE_DIR/ssh_key" +VIRSH=(virsh --connect qemu:///system) +SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5) + +log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; } +die() { + printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2 + exit 1 +} + +usage() { + sed -n '2,11p' "$0" | sed 's/^# \{0,1\}//' + exit 1 +} + +# delete_vm : stops and deletes a VM with its disk and seed ISO. +delete_vm() { + if "${VIRSH[@]}" dominfo "$1" >/dev/null 2>&1; then + "${VIRSH[@]}" destroy "$1" >/dev/null 2>&1 || true + "${VIRSH[@]}" undefine "$1" --nvram >/dev/null 2>&1 || "${VIRSH[@]}" undefine "$1" >/dev/null + fi + rm -f "$IMAGE_DIR/$1.qcow2" "$IMAGE_DIR/$1-seed.iso" +} + +vm_ip() { + "${VIRSH[@]}" domifaddr "$1" --source lease 2>/dev/null | + awk '$3 == "ipv4" { sub(/\/.*/, "", $4); print $4; exit }' +} + +wait_for_ip() { + local ip="" + for _ in $(seq 1 90); do + ip=$(vm_ip "$1") + [ -n "$ip" ] && break + sleep 2 + done + [ -n "$ip" ] || die "VM $1 did not get an IP address from DHCP." + echo "$ip" +} + +# wait_for_ssh : waits for SSH and for cloud-init to finish. +wait_for_ssh() { + for _ in $(seq 1 90); do + if ssh "${SSH_OPTS[@]}" "root@$1" true 2>/dev/null; then + ssh "${SSH_OPTS[@]}" "root@$1" 'cloud-init status --wait >/dev/null || true' + return 0 + fi + sleep 2 + done + die "SSH to root@$1 does not respond." +} + +vm_ssh() { ssh "${SSH_OPTS[@]}" "root@$VM_IP" "$@"; } + +ensure_offline_network() { + if ! "${VIRSH[@]}" net-info "$OFFLINE_NETWORK" >/dev/null 2>&1; then + log "Creating offline libvirt network $OFFLINE_NETWORK ($OFFLINE_SUBNET.0/24)" + local xml + xml=$(mktemp) + # No element: libvirt blocks all traffic between this network and other networks. + # : the VM cannot resolve host names through the host either. + cat >"$xml" < + $OFFLINE_NETWORK + + + + + + +EOF + "${VIRSH[@]}" net-define "$xml" >/dev/null + rm -f "$xml" + "${VIRSH[@]}" net-autostart "$OFFLINE_NETWORK" >/dev/null + fi + if [[ " $("${VIRSH[@]}" net-list --name | xargs) " != *" $OFFLINE_NETWORK "* ]]; then + "${VIRSH[@]}" net-start "$OFFLINE_NETWORK" >/dev/null + fi +} + +# create_vm : creates a VM with a copy-on-write disk on the image. +create_vm() { + local name=$1 backing=$2 network=$3 seed_dir + log "Creating VM $name on network $network (${VM_VCPUS} vCPU, ${VM_MEMORY} MiB RAM)" + qemu-img create -q -f qcow2 -F qcow2 -b "$backing" "$IMAGE_DIR/$name.qcow2" "$VM_DISK_SIZE" + + seed_dir=$(mktemp -d) + cat >"$seed_dir/user-data" <"$seed_dir/meta-data" + if command -v xorriso >/dev/null; then + xorriso -as mkisofs -quiet -V cidata -J -r -o "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data" 2>/dev/null + elif command -v cloud-localds >/dev/null; then + cloud-localds "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data" + else + genisoimage -quiet -V cidata -J -r -o "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data" + fi + rm -rf "$seed_dir" + + virt-install --connect qemu:///system --name "$name" --memory "$VM_MEMORY" --vcpus "$VM_VCPUS" \ + --import --os-variant ubuntu24.04 \ + --disk "path=$IMAGE_DIR/$name.qcow2,format=qcow2,bus=virtio" \ + --disk "path=$IMAGE_DIR/$name-seed.iso,device=cdrom" \ + --network "network=$network,model=virtio,mac=$VM_MAC" \ + --graphics none --noautoconsole >/dev/null +} + +# build_base_image : installs Coolify in a builder VM with internet access and keeps its disk. +build_base_image() { + local base=$1 + if [ ! -f "$UBUNTU_IMAGE" ]; then + log "Downloading Ubuntu 24.04 cloud image" + curl -fL --progress-bar -o "$UBUNTU_IMAGE.part" "$UBUNTU_IMAGE_URL" + mv "$UBUNTU_IMAGE.part" "$UBUNTU_IMAGE" + fi + + delete_vm "$BUILDER_NAME" + create_vm "$BUILDER_NAME" "$UBUNTU_IMAGE" "$NAT_NETWORK" + VM_IP=$(wait_for_ip "$BUILDER_NAME") + wait_for_ssh "$VM_IP" + + log "Installing Coolify $VERSION in the builder VM (no backup data in this VM)" + vm_ssh "curl -fsSL https://cdn.coollabs.io/coolify/install.sh | bash -s -- $VERSION" + + vm_ssh 'shutdown -h now' || true + for _ in $(seq 1 60); do + [ "$("${VIRSH[@]}" domstate "$BUILDER_NAME")" = "shut off" ] && break + sleep 2 + done + [ "$("${VIRSH[@]}" domstate "$BUILDER_NAME")" = "shut off" ] || die "The builder VM did not shut down." + "${VIRSH[@]}" undefine "$BUILDER_NAME" --nvram >/dev/null 2>&1 || "${VIRSH[@]}" undefine "$BUILDER_NAME" >/dev/null + mv "$IMAGE_DIR/$BUILDER_NAME.qcow2" "$base" + rm -f "$IMAGE_DIR/$BUILDER_NAME-seed.iso" +} + +# --- main --------------------------------------------------------------------- + +[ "${1:-}" = "destroy" ] && { + delete_vm "$VM_NAME" + delete_vm "$BUILDER_NAME" + rm -f "$IMAGE_DIR"/coolify-base-*.qcow2 + echo "Deleted VM $VM_NAME, its disk, and the Coolify base images." + exit 0 +} +[ $# -eq 0 ] || usage +[ -t 0 ] || die "Run this script in an interactive terminal: it asks for its inputs." + +read -r -e -p "Backup file (.dmp or .gz): " BACKUP_FILE +BACKUP_FILE=${BACKUP_FILE/#\~/$HOME} +read -r -s -p "APP_KEY of the backed-up instance (input is hidden): " APP_KEY +echo +read -r -p "Coolify version or image tag (empty for latest stable): " VERSION + +[ -s "$BACKUP_FILE" ] || die "Backup file '$BACKUP_FILE' does not exist or is empty." +[[ "$APP_KEY" == base64:* ]] || die "APP_KEY must start with 'base64:'." +for cmd in virsh virt-install qemu-img ssh scp ssh-keygen curl jq; do + command -v "$cmd" >/dev/null || die "Missing command: $cmd" +done +[ -e /dev/kvm ] || die "/dev/kvm is not available." + +if [ -z "$VERSION" ]; then + VERSION=$(curl -fsSL https://cdn.coollabs.io/coolify/versions.json | jq -r '.coolify.v4.version') + echo "Latest stable version: $VERSION" +fi +[[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] || die "Version '$VERSION' contains characters that are not allowed." +BASE_IMAGE="$IMAGE_DIR/coolify-base-$VERSION.qcow2" + +# Other VMs and containers can use most of the host memory. A VM that does not fit +# makes the host swap until it stops responding, so stop before it starts. +# The previous VM is deleted below, so its memory counts as available. +OLD_VM_KIB=$("${VIRSH[@]}" dominfo "$VM_NAME" 2>/dev/null | awk '/^State:/ { running = ($2 == "running") } /^Used memory:/ { kib = $3 } END { print running ? kib : 0 }' || true) +AVAILABLE_MIB=$(($(awk '/^MemAvailable:/ { print $2 }' /proc/meminfo) / 1024 + ${OLD_VM_KIB:-0} / 1024)) +REQUIRED_MIB=$((VM_MEMORY + 2048)) +if [ "$AVAILABLE_MIB" -lt "$REQUIRED_MIB" ]; then + die "Not enough free memory: ${AVAILABLE_MIB} MiB available, ${REQUIRED_MIB} MiB required (VM_MEMORY + 2048 MiB for the host). +Stop other VMs (virsh --connect qemu:///system list) or set a lower VM_MEMORY." +fi + +if "${VIRSH[@]}" dominfo "$VM_NAME" >/dev/null 2>&1; then + log "Deleting the previous VM $VM_NAME" +fi +delete_vm "$VM_NAME" + +mkdir -p "$IMAGE_DIR" +if [ ! -f "$SSH_KEY" ]; then + # The comment must not contain "coolify": install.sh removes such lines from authorized_keys. + ssh-keygen -q -t ed25519 -N '' -C restore-test-vm -f "$SSH_KEY" +fi + +if [ -f "$BASE_IMAGE" ]; then + log "Using the cached base image for Coolify $VERSION" +else + build_base_image "$BASE_IMAGE" +fi + +ensure_offline_network +create_vm "$VM_NAME" "$BASE_IMAGE" "$OFFLINE_NETWORK" +VM_IP=$(wait_for_ip "$VM_NAME") +log "VM IP address on the offline network: $VM_IP" +wait_for_ssh "$VM_IP" + +log "Confirming that the VM has no internet access" +if vm_ssh 'timeout 5 bash -c "echo >/dev/tcp/1.1.1.1/443" 2>/dev/null || timeout 5 bash -c "echo >/dev/tcp/8.8.8.8/53" 2>/dev/null'; then + die "The VM can reach the internet. Stopped before the backup was copied." +fi +if vm_ssh 'getent hosts cdn.coollabs.io >/dev/null'; then + die "The VM can resolve internet host names. Stopped before the backup was copied." +fi + +log "Copying the backup into the VM" +if gzip -t "$BACKUP_FILE" 2>/dev/null; then + gzip -dc "$BACKUP_FILE" | vm_ssh 'cat >/root/coolify-backup.dmp' +else + scp "${SSH_OPTS[@]}" "$BACKUP_FILE" "root@$VM_IP:/root/coolify-backup.dmp" +fi + +log "Restoring the backup and starting Coolify (migrations run on start)" +# Send APP_KEY through stdin, so it does not show in the process list of this host or the VM. +{ + printf 'APP_KEY=%q\n' "$APP_KEY" + cat <<'REMOTE' +set -euo pipefail +cd /data/coolify/source + +for _ in $(seq 1 60); do + [ "$(docker inspect -f '{{.State.Health.Status}}' coolify-db 2>/dev/null)" = healthy ] && break + sleep 2 +done + +# Start the same image tag again after the restore. +LATEST_IMAGE=$(docker inspect -f '{{.Config.Image}}' coolify | sed 's/.*://') +docker stop coolify coolify-realtime >/dev/null + +sed -i "s|^APP_KEY=.*|APP_KEY=${APP_KEY}|" .env + +# Turn off Horizon (queue workers) and the scheduler: no job from the backup runs. +# Jobs that the UI dispatches stay in Redis and are never processed. +for flag in HORIZON_ENABLED SCHEDULER_ENABLED; do + sed -i "/^${flag}=/d" .env + echo "${flag}=false" >>.env +done + +# Recreate the database: tables from a newer fresh install must not remain. +docker exec coolify-db psql -q -U coolify -d postgres -v ON_ERROR_STOP=1 \ + -c 'DROP DATABASE coolify WITH (FORCE)' -c 'CREATE DATABASE coolify OWNER coolify' +echo "Running pg_restore..." +docker exec -i coolify-db pg_restore --exit-on-error --no-acl --no-owner -U coolify -d coolify &1 | sed -n '/Migration is enabled/,/Seeder is enabled/p' +echo +echo "--- Errors after start ---" +docker logs coolify 2>&1 | grep -iE 'error|exception|fail' | tail -n 30 || true +echo +echo "--- Pending migrations ---" +docker exec coolify php artisan migrate:status --pending || true +echo +echo "--- Queue workers and scheduler ---" +if docker exec coolify sh -c "ps aux | grep -qE '[a]rtisan (horizon|schedule:work|queue:work)'"; then + echo "ERROR: A queue worker or the scheduler runs." + exit 1 +fi +echo "Horizon and the scheduler are off." +REMOTE +} | vm_ssh 'bash -s' + +HOST_ADDR=$(hostname -f 2>/dev/null || hostname) +cat <set('app.maintenance.store', 'array'); + config()->set('api.rate_limit', 2); + + InstanceSettings::forceCreate(['id' => 0]); +}); + +function pushSentinelFrom(string $clientIp): TestResponse +{ + return test() + ->withHeader('CF-Connecting-IP', $clientIp) + ->withServerVariables(['REMOTE_ADDR' => '172.70.0.1']) + ->postJson('/api/v1/sentinel/push'); +} + +it('gives each Cloudflare client ip its own sentinel push bucket on cloud', function () { + config()->set('constants.coolify.self_hosted', false); + + pushSentinelFrom('203.0.113.10')->assertUnauthorized(); + pushSentinelFrom('203.0.113.10')->assertUnauthorized(); + pushSentinelFrom('203.0.113.10')->assertStatus(429); + + pushSentinelFrom('203.0.113.20')->assertUnauthorized(); +}); + +it('ignores the Cloudflare client ip for sentinel pushes on self-hosted instances', function () { + config()->set('constants.coolify.self_hosted', true); + + pushSentinelFrom('203.0.113.10')->assertUnauthorized(); + pushSentinelFrom('203.0.113.10')->assertUnauthorized(); + + pushSentinelFrom('203.0.113.20')->assertStatus(429); +}); + +it('keys unauthenticated limiters by the Cloudflare client ip on cloud', function (string $limiter) { + config()->set('constants.coolify.self_hosted', false); + + $request = Request::create('/', server: [ + 'REMOTE_ADDR' => '172.70.0.1', + 'HTTP_CF_CONNECTING_IP' => '203.0.113.10', + ]); + + $limit = RateLimiter::limiter($limiter)($request); + + expect($limit->key)->toBe('203.0.113.10'); +})->with(['api', '5', 'feedback']);