Merge remote-tracking branch 'origin/main'

scripts/dev

@@ -24,12 +24,17 @@
 #
 # Env: COOLIFY_DEV_SERVER_BACKEND=auto|lima|testing-host, COOLIFY_DEV_KVM_PROFILE=<profile>
 #
+# Side by side: COOLIFY_DEV_INSTANCE=<name> COOLIFY_DEV_SIDE_BY_SIDE=1 starts a second instance of
+# this checkout without stopping the other ones (for example a transfer target, see
+# scripts/transfer-test). It gets its own SSH key folder; the instance remembers the mode.
+#
 set -euo pipefail
 
 cd "$(dirname "$0")/.."
 ROOT="$(pwd -P)"
 
 COMPOSE_FILE="docker-compose.dev-multi.yml"
+SIDE_BY_SIDE_COMPOSE_FILE="docker-compose.dev-side-by-side.yml"
 GIT_COMMON_DIR="$(git rev-parse --path-format=absolute --git-common-dir)"
 # Shared by all worktrees, so slots, APP_KEYs, and data survive worktree removal.
 ENV_DIR="$(dirname "$GIT_COMMON_DIR")/.dev-instances"
@@ -38,7 +43,7 @@ QEMU_STORAGE="${DEVELOPMENT_QEMU_STORAGE_PATH:-/var/lib/libvirt/images/coolify-d
 LEGACY_CONTAINERS=(coolify coolify-db coolify-redis coolify-vite coolify-mail coolify-minio coolify-minio-init coolify-testing-host)
 
 usage() {
-  sed -n '2,26p' "$0" | sed 's/^# \?//'
+  sed -n '2,30p' "$0" | sed 's/^# \?//'
 }
 
 normalize_name() {
@@ -66,6 +71,7 @@ current_instance() {
 project_name() { echo "coolify-dev-$1"; }
 env_file() { echo "$ENV_DIR/$1.env"; }
 env_value() { grep -E "^$2=" "$(env_file "$1")" | cut -d= -f2-; }
+is_side_by_side() { [[ -f "$(env_file "$1")" ]] && grep -qxF 'DEV_SIDE_BY_SIDE=1' "$(env_file "$1")"; }
 
 # Lowest free slot (1..254), stable per instance name. It selects the port block
 # and the libvirt network (10.221.<slot>.0/24) of the instance.
@@ -135,6 +141,11 @@ ensure_env() {
   fi
   printf '%s\n' "$existing_key" >"$ENV_DIR/$name.appkey"
 
+  local side_by_side="${COOLIFY_DEV_SIDE_BY_SIDE:-}"
+  if [[ -z "$side_by_side" ]] && is_side_by_side "$name"; then
+    side_by_side=1
+  fi
+
   cat >"$env_file" <<EOF
 # Generated by scripts/dev for instance ${name} — do not commit
 COMPOSE_PROJECT_NAME=$(project_name "$name")
@@ -158,6 +169,9 @@ PUSHER_APP_ID=coolify-${name}
 PUSHER_APP_KEY=coolify-${name}
 PUSHER_APP_SECRET=coolify-${name}
 EOF
+  if [[ "$side_by_side" == 1 ]]; then
+    echo 'DEV_SIDE_BY_SIDE=1' >>"$env_file"
+  fi
 }
 
 # compose <name> <args...>: .env of the checkout first, generated instance env wins.
@@ -168,7 +182,11 @@ compose() {
   if [[ -f .env ]]; then
     env_files+=(--env-file .env)
   fi
-  docker compose -p "$(project_name "$name")" -f "$COMPOSE_FILE" \
+  local -a compose_files=(-f "$COMPOSE_FILE")
+  if is_side_by_side "$name"; then
+    compose_files+=(-f "$SIDE_BY_SIDE_COMPOSE_FILE")
+  fi
+  docker compose -p "$(project_name "$name")" "${compose_files[@]}" \
     "${env_files[@]}" --env-file "$(env_file "$name")" "${profiles[@]}" "$@"
 }
 
@@ -221,12 +239,14 @@ wait_for_coolify() {
 
 # Only one instance of a checkout runs at a time: they share its ports (main checkout)
 # and its public/hot Vite file. The legacy fixed-name stack also holds the main ports.
+# Side-by-side instances have their own ports and run next to the others on purpose.
 stop_other_checkout_instances() {
   local file other
+  is_side_by_side "$instance" && return 0
   for file in "$ENV_DIR"/*.env; do
     [[ -f "$file" ]] || continue
     other="$(basename "$file" .env)"
-    if [[ "$other" != "$instance" ]] && grep -qxF "DEV_CHECKOUT=$ROOT" "$file" &&
+    if [[ "$other" != "$instance" ]] && ! is_side_by_side "$other" && grep -qxF "DEV_CHECKOUT=$ROOT" "$file" &&
       [[ -n "$(docker ps -q --filter "label=com.docker.compose.project=$(project_name "$other")")" ]]; then
       echo "Stopping instance ${other} (same checkout)"
       docker compose -p "$(project_name "$other")" stop
@@ -382,9 +402,10 @@ start_backend() {
     local host="${DEV_BIND_ADDRESS:-127.0.0.1}"
     [[ "$host" == 0.0.0.0 ]] && host=127.0.0.1
     echo "Using KVM profile: $profile"
-    # The host command must reach this instance's database and encrypt with this instance's
-    # APP_KEY, not the ones in .env, and must not create root-owned files in storage/logs.
-    LOG_CHANNEL=stderr APP_KEY="$(env_value "$instance" APP_KEY)" \
+    # Use this instance database and encryption key, without a stale environment or config cache.
+    # Send logs to stderr to avoid root-owned files in storage/logs.
+    APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" \
+      LOG_CHANNEL=stderr APP_KEY="$(env_value "$instance" APP_KEY)" \
       DB_HOST="$host" DB_PORT="$(env_value "$instance" FORWARD_DB_PORT)" \
       REDIS_HOST="$host" REDIS_PORT="$(env_value "$instance" FORWARD_REDIS_PORT)" \
       DEVELOPMENT_QEMU_INSTANCE="$instance" \

scripts/patch.sh

@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+# Patches Coolify 4.0 to 4.4 to the security fix release of its own line.
+# Only the coolify container is recreated, the compose files and the other containers stay as they are.
+# Usage: curl -fsSL https://cdn.coollabs.io/coolify/patch.sh | sudo bash
+set -euo pipefail
+
+SOURCE_DIR=/data/coolify/source
+ENV_FILE="$SOURCE_DIR/.env"
+TIMESTAMP="$(date +%Y-%m-%d-%H-%M-%S)"
+LOG_FILE="$SOURCE_DIR/patch-$TIMESTAMP.log"
+BACKUP_FILE="$ENV_FILE-$TIMESTAMP"
+PATCH_VERSIONS=(4.0.1 4.1.3 4.2.1 4.3.24 4.4.7)
+
+BOLD=$'\033[1m' GREEN=$'\033[32m' BLUE=$'\033[34m' RED=$'\033[31m' RESET=$'\033[0m'
+
+heading() { printf '\n  %s%s%s\n' "$BOLD" "$*" "$RESET"; }
+info() { printf '  %sℹ%s %s\n' "$BLUE" "$RESET" "$*"; }
+ok() { printf '  %s✔%s %s\n' "$GREEN" "$RESET" "$*"; }
+die() {
+	printf '  %s✖%s %s\n' "$RED" "$RESET" "$*" >&2
+	exit 1
+}
+
+main() {
+	check_installation
+	read_version
+	select_patch_version
+	check_compose_files
+	heading "Patching Coolify $current_version to $patch_version"
+	pull_image
+	update_env_file
+	restart_coolify
+	wait_for_coolify
+}
+
+check_installation() {
+	heading "Checking the installation"
+	[ "$EUID" = 0 ] || die "run this script as root, for example with sudo"
+	[ -f "$ENV_FILE" ] && [ -f "$SOURCE_DIR/docker-compose.prod.yml" ] || die "Coolify is not installed in $SOURCE_DIR"
+	docker compose version > /dev/null 2>&1 || die "the Docker Compose plugin is not installed"
+	compose=(docker compose --env-file "$ENV_FILE" -f "$SOURCE_DIR/docker-compose.yml" -f "$SOURCE_DIR/docker-compose.prod.yml")
+	if [ -f "$SOURCE_DIR/docker-compose.custom.yml" ]; then
+		compose+=(-f "$SOURCE_DIR/docker-compose.custom.yml")
+	fi
+	if [ -f "$SOURCE_DIR/docker-compose.postgres-upgrade.yml" ]; then
+		compose+=(-f "$SOURCE_DIR/docker-compose.postgres-upgrade.yml")
+	fi
+	ok "Coolify is installed in $SOURCE_DIR"
+}
+
+read_version() {
+	# Copy the file out of the container, because this also works when the mitigation advice was followed and the container is stopped
+	current_version="$(docker cp coolify:/var/www/html/config/constants.php - 2> /dev/null | tar -xO 2> /dev/null | awk -F "'" "/'version' =>/ && !found { print \$(NF - 1); found = 1 }")" && [ -n "$current_version" ] || die "could not read the Coolify version from the coolify container"
+	ok "Coolify $BOLD$current_version$RESET is installed"
+}
+
+select_patch_version() {
+	[[ "$current_version" =~ ^4\.([0-4])\.([0-9]+)$ ]] || die "Coolify $current_version has no security fix release, upgrade to the latest version from the dashboard instead"
+	patch_version="${PATCH_VERSIONS[${BASH_REMATCH[1]}]}"
+	if [ "${BASH_REMATCH[2]}" -ge "${patch_version##*.}" ]; then
+		ok "Coolify $current_version already includes the security fixes"
+		exit 0
+	fi
+}
+
+check_compose_files() {
+	local has_realtime=false needs_realtime=true
+	if grep -q 'coollabsio/coolify-realtime:' "$SOURCE_DIR/docker-compose.prod.yml"; then
+		has_realtime=true
+	fi
+	if [[ "$patch_version" == 4.4.* ]]; then
+		needs_realtime=false
+	fi
+	[ "$has_realtime" = "$needs_realtime" ] || die "the compose files in $SOURCE_DIR do not belong to Coolify $current_version, update to ${PATCH_VERSIONS[4]} instead: curl -fsSL https://cdn.coollabs.io/coolify/upgrade.sh | sudo bash -s ${PATCH_VERSIONS[4]}"
+	ok "the compose files belong to Coolify $current_version"
+}
+
+pull_image() {
+	LATEST_IMAGE="$patch_version" "${compose[@]}" pull coolify < /dev/null >> "$LOG_FILE" 2>&1 || die "could not pull Coolify $patch_version, nothing was changed (log: $LOG_FILE)"
+	ok "pulled Coolify $patch_version"
+}
+
+update_env_file() {
+	local key updated
+	cp "$ENV_FILE" "$BACKUP_FILE"
+	for key in LATEST_IMAGE COOLIFY_VERSION; do
+		if grep -q "^$key=" "$ENV_FILE"; then
+			updated="$(sed "s|^$key=.*|$key=$patch_version|" "$ENV_FILE")"
+			printf '%s\n' "$updated" > "$ENV_FILE"
+		fi
+	done
+	ok "backed up .env to $BACKUP_FILE"
+}
+
+restart_coolify() {
+	# Detached, because the Coolify 4.4 terminal runs inside the coolify container and closes when it stops
+	LATEST_IMAGE="$patch_version" nohup "${compose[@]}" up -d --no-deps coolify < /dev/null >> "$LOG_FILE" 2>&1 &
+	wait $! || die "could not restart the coolify container (log: $LOG_FILE)"
+	ok "restarted the coolify container"
+}
+
+wait_for_coolify() {
+	local status
+	for _ in $(seq 60); do
+		status="$(docker inspect --format '{{.State.Health.Status}}' coolify 2> /dev/null || true)"
+		if [ "$status" = healthy ]; then
+			ok "Coolify $BOLD$patch_version$RESET is healthy and runs PHP $(docker exec coolify php -r 'echo PHP_VERSION;')"
+			return
+		fi
+		sleep 5
+	done
+	info "roll back with: cp $BACKUP_FILE $ENV_FILE && LATEST_IMAGE=$current_version ${compose[*]} up -d --no-deps coolify"
+	die "Coolify $patch_version is not healthy after 5 minutes, check it with: docker logs coolify"
+}
+
+main

scripts/transfer-test/README.md

@@ -0,0 +1,70 @@
+# Server transfer test kit
+
+Live end-to-end tests for the server transfer (moving a server and its resources to another Coolify instance).
+The unit tests in `tests/Unit/ServerTransfer/` fake SSH and HTTP. These scripts use two real dev instances and
+real VMs, so they also find problems with shells, sudo, networks, queues, and running containers.
+
+- **Source (A):** the dev instance of this checkout (`./scripts/dev start`). Its localhost is the `ubuntu-root` VM.
+  The tests transfer its `ubuntu-non-root` and `debian-root` VMs.
+- **Target (B):** a second instance of the same checkout that runs side by side (`transfer-target`). Its localhost is
+  the testing-host container.
+
+Requirements: Linux with KVM (the dev VMs), root or passwordless sudo (VMs and iptables), `jq`, and PHP on the host.
+macOS with Lima is not supported yet.
+
+## Steps
+
+```bash
+./scripts/dev start                        # A, if it does not run yet
+scripts/transfer-test/setup.sh             # start B and connect A, B, and A's VMs (safe to run again)
+scripts/transfer-test/reset.sh             # WIPES BOTH DEV DATABASES, then seeds A with deployed resources (~4 min)
+
+scripts/transfer-test/e2e.sh               # transfer both VMs A -> B through the API (41 checks)
+scripts/transfer-test/reset.sh
+scripts/transfer-test/reverse-e2e.sh       # undo paths: transfer back, transfer over an archive, take back, archive delete (18 checks)
+scripts/transfer-test/reset.sh
+scripts/transfer-test/extras-e2e.sh        # fake tokens, DNS, secret links, runner config, links, icon ... (31 checks)
+scripts/transfer-test/reset.sh
+```
+
+For a manual test, run `scripts/transfer-test/manual-setup.sh`: it resets, adds the fake extra records to A, and
+prints the URLs, logins, and tokens.
+
+Each test ends with `=== N failure(s)` and exits with N. Run `reset.sh` between two tests: a test leaves servers
+transferred. After `reset.sh`, you can also test by hand: log in to A and B with `test@example.com` / `password`, open a
+VM server on A → Transfer, and use the target URL `http://localhost:<B port>` with the token in
+`<main checkout>/.dev-instances/transfer-test/<source>/target.token`.
+
+## Files
+
+| File | What it does |
+|---|---|
+| `lib.sh` | Shared values (from `.dev-instances/*.env` and `config/development-qemu.php`) and helpers |
+| `setup.sh` | Starts B side by side, adds the iptables rule for B → A's VMs, and adds socat forwards when the ports listen on 127.0.0.1 |
+| `reset.sh` | Removes test workloads from the VMs, wipes both databases, seeds A's VMs, makes the helper image available, creates API tokens, runs `seed.sh` |
+| `seed.sh` | Creates and deploys resources on A through the API |
+| `e2e.sh`, `reverse-e2e.sh`, `extras-e2e.sh` | The tests |
+| `seed-extras.php`, `verify-extras-*.php` | Fake records and checks of `extras-e2e.sh` |
+| `manual-setup.sh` | Reset plus the fake extra records, then prints what to open and enter for a manual test |
+| `seed-inventory.php` | Seeds a large database-only demo server (no deploys) on the current instance |
+
+Tokens and the VM SSH key are written to `<main checkout>/.dev-instances/transfer-test/<source>/` (not in git).
+For a full manual test of every transferred setting, see `docs/server-transfer-full-coverage.md`.
+
+## Options
+
+- `TRANSFER_TEST_SOURCE=<instance>`: another source instance than the one of this checkout.
+- `TRANSFER_TEST_TARGET=<instance>`: another target name than `transfer-target`.
+
+## Notes
+
+- A and B share `storage/` of the checkout. The side-by-side mode of `scripts/dev` gives B its own SSH folder, because
+  both instances have a server with the uuid `localhost`.
+- A and B have the same `app.id`, so the DNS owner comment does not change in a transfer here.
+  `extras-e2e.sh` expects that. With different ids, the target must change the comment at the DNS provider.
+- `reset.sh` tags the newest local helper image with the configured helper version when that version is not
+  published yet.
+- Remove the kit: run `./scripts/dev destroy transfer-target` and
+  `docker rm -f $(docker ps -aq --filter name=coolify-transfer-test-forward)`. Then remove the iptables rule that
+  `setup.sh` added: `sudo iptables -D LIBVIRT_FWI -s <B subnet> -d 10.221.<A slot>.0/24 -o cdevbr<A slot> -j ACCEPT`
+  (`setup.sh` prints the subnet; `sudo iptables -S LIBVIRT_FWI` lists the rules).

scripts/transfer-test/e2e.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Transfer both VM servers A -> B through the API and check source, target, server, and a redeploy from B.
+# Run reset.sh before and after.
+source "$(dirname "$0")/lib.sh"
+TA="$(source_token)"
+TB="$(target_token)"
+
+for profile in "${TRANSFER_PROFILES[@]}"; do
+  S="$(vm_field "$profile" uuid)"
+  ip="$(vm_field "$profile" ip)"
+  echo "=== $S"
+  before="$(stable_workloads "$profile")"
+  apps="$(tinker "$A" "echo App\\Models\\Server::where('uuid','$S')->first()->applications()->pluck('uuid')->implode(' ');" | tail -1)"
+
+  echo '-- start the transfer on A'
+  r="$(api POST "$A_PORT" "/servers/$S/migrate" "$TA" "$(jq -nc --arg u "http://localhost:$B_PORT" --arg t "$TB" '{target_url:$u,target_token:$t}')")"
+  if [[ "$(jq -r .migration.status <<<"$r" 2>/dev/null)" != running ]]; then
+    bad "transfer request accepted: $r"
+    continue
+  fi
+  ok 'transfer request accepted'
+  for _ in $(seq 1 40); do sleep 3; st="$(api GET "$A_PORT" "/servers/$S/transfer" "$TA" | jq -r .migration.status)"; [[ "$st" != running ]] && break; done
+  check "transfer finished ($st)" test "$st" = succeeded
+
+  echo '-- source A'
+  a="$(api GET "$A_PORT" "/servers/$S/transfer" "$TA")"
+  export_id="$(jq -r .transfer.export_id <<<"$a")"
+  check 'A transfer.status = transferred' test "$(jq -r .transfer.status <<<"$a")" = transferred
+  check 'A force_disabled + Sentinel off' test "$(tinker "$A" "\$s=App\\Models\\Server::where('uuid','$S')->first()->settings; echo (int)\$s->force_disabled.(int)\$s->is_sentinel_enabled;" | tail -1)" = 10
+  check 'A shows the take-back modal' test "$(tinker "$A" "$login_php echo (int) str_contains(Livewire\\Livewire::test(App\\Livewire\\Server\\Show::class, ['server_uuid'=>'$S'])->html(), 'Take back a transferred server?');" | tail -1)" = 1
+  r="$(api POST "$A_PORT" "/servers/$S/migrate" "$TA" "$(jq -nc --arg u "http://localhost:$B_PORT" --arg t "$TB" '{target_url:$u,target_token:$t}')")"
+  check 'second transfer refused by the prechecks' grep -q 'already transferred' <<<"$r"
+
+  echo '-- target B'
+  b="$(api GET "$B_PORT" "/servers/$S/transfer" "$TB")"
+  check 'B transfer.status = claimed' test "$(jq -r .transfer.status <<<"$b")" = claimed
+  check 'same export id on A and B' test "$(jq -r .transfer.export_id <<<"$b")" = "$export_id"
+  for _ in $(seq 1 20); do
+    sleep 3
+    meta="$(tinker "$B" "\$s=App\\Models\\Server::where('uuid','$S')->first(); echo json_encode(['os'=>data_get(\$s->server_metadata,'os'),'reach'=>\$s->settings->is_reachable,'usable'=>\$s->settings->is_usable,'logs'=>\$s->validation_logs,'transfer'=>data_get(\$s->server_metadata,'transfer.status')]);" | tail -1)"
+    [[ "$(jq -r .os <<<"$meta")" != null ]] && break
+  done
+  check "automatic validation filled the metadata ($(jq -r .os <<<"$meta"))" test "$(jq -r .os <<<"$meta")" != null
+  check 'B reachable + usable, no validation error' test "$(jq -c '[.reach,.usable,.logs]' <<<"$meta")" = '[true,true,null]'
+  check 'transfer state kept after validation' test "$(jq -r .transfer <<<"$meta")" = claimed
+  check 'B started the finish list' test "$(tinker "$B" "echo count((array) data_get(App\\Models\\Server::where('uuid','$S')->first()->server_metadata,'transfer.finish.items')) > 0 ? 1 : 0;" | tail -1)" = 1
+  check 'A links to the server on B' test "$(jq -r .migration.target_server_url <<<"$a")" = "$B_URL/server/$S"
+  check 'B has all applications' test "$(tinker "$B" "echo App\\Models\\Server::where('uuid','$S')->first()->applications()->pluck('uuid')->implode(' ');" | tail -1)" = "$apps"
+  check 'B validates the server over SSH' test "$(tinker "$B" "\$r=App\\Models\\Server::where('uuid','$S')->first()->validateConnection(); echo (int)(bool)\$r['uptime'];" | tail -1)" = 1
+
+  echo '-- server'
+  sleep 5
+  after="$(workloads "$profile")"
+  check 'proxy + workloads not recreated' test "$before" = "$after"
+  sentinel_env="$(ssh_vm "$profile" "docker inspect coolify-sentinel --format '{{range .Config.Env}}{{println .}}{{end}}'")"
+  check 'Sentinel sends to B' grep -qx "PUSH_ENDPOINT=$B_URL" <<<"$sentinel_env"
+  for app in $apps; do check "app $app answers over HTTP" bash -c "curl -sf -m5 -H 'Host: $app.$ip.sslip.io' http://$ip/ >/dev/null"; done
+
+  echo '-- redeploy one app from B'
+  app="${apps%% *}"
+  api POST "$B_PORT" "/deploy?uuid=$app&force=false" "$TB" >/dev/null
+  for _ in $(seq 1 40); do sleep 5; ds="$(tinker "$B" "echo App\\Models\\ApplicationDeploymentQueue::where('application_id', App\\Models\\Application::where('uuid','$app')->value('id'))->latest('id')->value('status');" | tail -1)"; [[ "$ds" == finished || "$ds" == failed ]] && break; done
+  check "redeploy from B ($ds)" test "$ds" = finished
+  check 'app answers after the redeploy' bash -c "for i in {1..10}; do curl -sf -m5 -H 'Host: $app.$ip.sslip.io' http://$ip/ >/dev/null && exit 0; sleep 3; done; exit 1"
+done
+
+echo "=== $FAILS failure(s)"
+exit "$FAILS"

scripts/transfer-test/extras-e2e.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Add fake records around debian-root on A (tokens, DNS, secret manager link, tunnel token, GitHub App + runner config,
+# resource links, canvas positions, team variable, maintenance mode, proxy error page, project icon, and an app of the
+# same project on the other VM), transfer debian-root to B through the Transfer page, and check them on B and A.
+# Run reset.sh before and after.
+source "$(dirname "$0")/lib.sh"
+S="$(vm_field debian-root uuid)"
+
+echo '=== seed fake records on A'
+php_file "$A" "$TT_DIR/seed-extras.php"
+echo "=== transfer $S A -> B"
+echo "status: $(transfer_from_page "$S")"
+
+# The target compares the DNS owner comment, which contains the instance id, with the one of the source.
+tinker "$A" 'echo config("app.id");' | tail -1 | docker exec -i "$B" sh -c 'cat > /tmp/source-app-id'
+
+echo '=== target B'
+b="$(php_file "$B" "$TT_DIR/verify-extras-target.php")"
+echo "$b"
+echo '=== source A'
+a="$(php_file "$A" "$TT_DIR/verify-extras-source.php")"
+echo "$a"
+FAILS="$(printf '%s\n%s\n' "$a" "$b" | grep -c '^FAIL')"
+echo "=== $FAILS failure(s)"
+exit "$FAILS"

scripts/transfer-test/lib.sh

@@ -0,0 +1,153 @@
+#!/usr/bin/env bash
+# Shared values and helpers of the server transfer test kit. The other scripts source this file.
+#
+# Source (A): the dev instance of this checkout, as `./scripts/dev` names it. Override: TRANSFER_TEST_SOURCE.
+# Target (B): a side-by-side instance of the same checkout. Default "transfer-target". Override: TRANSFER_TEST_TARGET.
+# Tokens and the VM SSH key are stored in <main checkout>/.dev-instances/transfer-test/<source>/ (not in git).
+
+set -uo pipefail
+
+TT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+ROOT="$(cd "$TT_DIR/../.." && pwd -P)"
+cd "$ROOT"
+ENV_DIR="$(dirname "$(git rev-parse --path-format=absolute --git-common-dir)")/.dev-instances"
+
+SOURCE="${TRANSFER_TEST_SOURCE:-$(./scripts/dev container | sed 's/^coolify-dev-//')}"
+TARGET="${TRANSFER_TEST_TARGET:-transfer-target}"
+A="coolify-dev-$SOURCE"
+B="coolify-dev-$TARGET"
+STATE_DIR="$ENV_DIR/transfer-test/$SOURCE"
+VM_KEY="$STATE_DIR/vm_key"
+
+# The VM servers that the tests transfer, and the profile that is localhost of the source.
+TRANSFER_PROFILES=(ubuntu-non-root debian-root)
+LOCALHOST_PROFILE=ubuntu-root
+
+PRIVILEGED=()
+if [[ $EUID -ne 0 ]]; then
+  PRIVILEGED=(sudo -n -E)
+fi
+
+die() {
+  echo "ERROR: $*" >&2
+  exit 1
+}
+
+env_value() {
+  local file="$ENV_DIR/$1.env"
+  [[ -f "$file" ]] || die "No dev instance '$1' ($file). See ./scripts/dev urls"
+  grep -E "^$2=" "$file" | cut -d= -f2-
+}
+
+A_PORT="$(env_value "$SOURCE" APP_PORT)"
+A_SLOT="$(env_value "$SOURCE" DEVELOPMENT_QEMU_SLOT)"
+A_URL="$(env_value "$SOURCE" APP_URL)"
+
+target_values() {
+  B_PORT="$(env_value "$TARGET" APP_PORT)"
+  B_URL="$(env_value "$TARGET" APP_URL)"
+}
+if [[ -f "$ENV_DIR/$TARGET.env" ]]; then
+  target_values
+fi
+
+# vm_field <profile> <field>: a value of config/development-qemu.php for the source instance (ip, user, uuid).
+vm_field() {
+  DEVELOPMENT_QEMU_INSTANCE="$SOURCE" DEVELOPMENT_QEMU_SLOT="$A_SLOT" php -r '
+    function env($key, $default = null) { $value = getenv($key); return $value === false ? $default : $value; }
+    $config = require "config/development-qemu.php";
+    echo $config["profiles"][$argv[1]][$argv[2]];
+  ' "$1" "$2"
+}
+
+running() { [[ "$(docker inspect --format '{{.State.Running}}' "$1" 2>/dev/null)" == true ]]; }
+
+# tinker <container> <php>: run PHP in a Coolify container.
+tinker() { docker exec -u www-data "$1" php artisan tinker --execute "$2" 2>/dev/null; }
+
+# php_file <container> <file>: run a PHP file of this kit in a Coolify container.
+php_file() {
+  docker cp "$2" "$1:/tmp/$(basename "$2")" >/dev/null &&
+    docker exec -u www-data "$1" php artisan tinker --execute "require '/tmp/$(basename "$2")';" 2>&1 | grep -v '^$'
+}
+
+# api <METHOD> <port> <path> <token> [json]
+api() {
+  curl -sS -m 30 -X "$1" "http://127.0.0.1:$2/api/v1$3" -H "Authorization: Bearer $4" \
+    -H 'Content-Type: application/json' -H 'Accept: application/json' ${5:+-d "$5"}
+}
+
+source_token() { cat "$STATE_DIR/source.token"; }
+target_token() { cat "$STATE_DIR/target.token"; }
+
+# The VMs accept the dev key that the seeders create as private key 1.
+fetch_vm_key() {
+  mkdir -p "$STATE_DIR"
+  tinker "$A" 'echo optional(App\Models\PrivateKey::find(1))->private_key;' >"$VM_KEY.tmp"
+  if grep -q 'PRIVATE KEY' "$VM_KEY.tmp"; then
+    mv "$VM_KEY.tmp" "$VM_KEY"
+    chmod 600 "$VM_KEY"
+    return 0
+  fi
+  rm -f "$VM_KEY.tmp"
+  return 1
+}
+
+# ssh_vm <profile> <command>: run a shell command as root on a VM of the source.
+ssh_vm() {
+  local user ip sudo=''
+  user="$(vm_field "$1" user)"
+  ip="$(vm_field "$1" ip)"
+  [[ "$user" != root ]] && sudo='sudo '
+  ssh -i "$VM_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 \
+    "$user@$ip" "${sudo}sh -c $(printf '%q' "$2")"
+}
+
+# workloads <profile>: the containers on a VM, without Sentinel, which a claim restarts on purpose.
+workloads() { ssh_vm "$1" 'docker ps --format "{{.Names}} {{.ID}}" | grep -v coolify-sentinel | sort'; }
+
+# stable_workloads <profile>: the workloads once they stop changing. reset.sh returns while databases still start.
+stable_workloads() {
+  local previous='' current
+  for _ in $(seq 1 30); do
+    current="$(workloads "$1")"
+    [[ -n "$current" && "$current" == "$previous" ]] && break
+    previous="$current"
+    sleep 10
+  done
+  echo "$current"
+}
+
+# require_dev_instance <container>: refuse to wipe anything that is not a running local dev instance.
+require_dev_instance() {
+  running "$1" || die "Container $1 is not running."
+  [[ "$1" == coolify-dev-* ]] || die "$1 is not a scripts/dev instance."
+  [[ "$(tinker "$1" 'echo app()->environment();' | tail -1)" == local ]] || die "$1 does not run with APP_ENV=local."
+}
+
+# Pass/fail counter of the e2e scripts.
+FAILS=0
+ok() { echo "  PASS $*"; }
+bad() {
+  echo "  FAIL $*"
+  FAILS=$((FAILS + 1))
+}
+check() {
+  local name="$1"
+  shift
+  if "$@" >/dev/null 2>&1; then ok "$name"; else bad "$name"; fi
+}
+
+login_php="auth()->login(App\\Models\\User::find(0)); session(['currentTeam' => App\\Models\\Team::find(0)]);"
+
+# transfer_from_page <server uuid>: start a transfer A -> B through the Transfer page, wait, print the final status.
+transfer_from_page() {
+  local status
+  tinker "$A" "$login_php Livewire\\Livewire::test(App\\Livewire\\Server\\Transfer::class, ['server_uuid' => '$1'])->set('targetUrl', 'http://localhost:$B_PORT')->set('targetToken', '$(target_token)')->call('migrateServer');" >/dev/null
+  for _ in $(seq 1 60); do
+    sleep 3
+    status="$(tinker "$A" "echo data_get(App\\Models\\Server::where('uuid', '$1')->first()?->server_metadata, 'migration.status');" | tail -1)"
+    [[ "$status" != running ]] && break
+  done
+  echo "$status"
+}

scripts/transfer-test/manual-setup.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Prepare A and B for a manual transfer test: reset, then add the fake extra records of extras-e2e.sh to A
+# (tokens, DNS, secret manager link, tunnel token, GitHub App + runner config, resource links, canvas positions,
+# team variable, maintenance mode, proxy error page, project icon, a git app with a system-wide GitHub App, and an
+# app of the same project on the other VM). Prints what to open and what to enter.
+#
+# Usage: scripts/transfer-test/manual-setup.sh
+source "$(dirname "$0")/lib.sh"
+
+"$TT_DIR/reset.sh" || die 'reset.sh failed'
+echo '== add the fake extra records to A'
+php_file "$A" "$TT_DIR/seed-extras.php"
+
+cat <<EOF
+
+Ready for a manual transfer test.
+
+  A (source): $A_URL
+  B (target): $B_URL
+  Login on both: test@example.com / password
+
+  Servers on A:
+    $(vm_field ubuntu-non-root uuid)  $(vm_field ubuntu-non-root ip)  project "Transfer Demo": whoami, nginx-dockerfile, postgres, redis, uptime-kuma
+    $(vm_field debian-root uuid)  $(vm_field debian-root ip)  project "Second Server": whoami-2, mariadb, git-shop, and all the fake extra records
+
+  Transfer on A: open a server -> Transfer
+    Target instance URL: http://localhost:$B_PORT
+    Target API token:    $(target_token)
+  Transfer back on A (archive -> Transfer):
+    Instance that manages the server now: http://localhost:$B_PORT   (same token)
+    URL of this instance:                 http://localhost:$A_PORT
+  Token of A (for a transfer from B to A): $(source_token)
+
+Reset again: scripts/transfer-test/manual-setup.sh
+EOF

scripts/transfer-test/reset.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# Put the kit back to its start: A manages its localhost VM and the transfer VMs with deployed resources, B is empty.
+# WIPES BOTH DEV DATABASES (migrate:fresh --seed) and the test workloads on the VMs. VM disks are kept. About 4 minutes.
+#
+# Usage: scripts/transfer-test/reset.sh
+source "$(dirname "$0")/lib.sh"
+
+require_dev_instance "$A"
+require_dev_instance "$B"
+mkdir -p "$STATE_DIR"
+
+echo '== 1. remove test workloads from the VMs (the proxy stays)'
+if fetch_vm_key; then
+  for profile in "${TRANSFER_PROFILES[@]}"; do
+    ssh_vm "$profile" '
+      for c in $(docker ps -aq); do n=$(docker inspect -f "{{.Name}}" $c); case "$n" in /coolify-proxy*) ;; *) docker rm -f $c >/dev/null;; esac; done
+      docker volume ls -q | grep -v "^coolify" | xargs -r docker volume rm >/dev/null 2>&1 || true
+      rm -rf /data/coolify/applications/* /data/coolify/databases/* /data/coolify/services/* /data/coolify/sentinel/* /data/coolify/transfer
+      echo "  $(hostname): $(docker ps --format "{{.Names}}" | tr "\n" " ")"' || echo "  $profile: not reachable, skipped"
+  done
+else
+  echo '  no VM key yet (first run), skipped'
+fi
+
+echo '== 2. fresh databases, Redis, SSH folders'
+for C in "$A" "$B"; do
+  docker exec "$C-redis-1" redis-cli FLUSHALL >/dev/null
+  docker exec "$C" sh -c 'rm -rf /var/www/html/storage/app/ssh/keys/* /var/www/html/storage/app/ssh/mux/*; mkdir -p /var/www/html/storage/app/ssh/keys /var/www/html/storage/app/ssh/mux; chown -R www-data:www-data /var/www/html/storage/app/ssh'
+  docker exec -u www-data "$C" php artisan migrate:fresh --seed --force >/dev/null || die "migrate:fresh failed in $C"
+  echo "  $C: database seeded"
+done
+
+echo '== 3. seed the VM servers of A'
+host="${DEV_BIND_ADDRESS:-$(grep -E '^DEV_BIND_ADDRESS=' .env 2>/dev/null | tail -1 | cut -d= -f2-)}"
+[[ -z "$host" || "$host" == 0.0.0.0 ]] && host=127.0.0.1
+APP_ENV=local APP_CONFIG_CACHE="$ROOT/bootstrap/cache/dev-qemu-$$.php" LOG_CHANNEL=stderr \
+  APP_KEY="$(env_value "$SOURCE" APP_KEY)" DB_HOST="$host" DB_PORT="$(env_value "$SOURCE" FORWARD_DB_PORT)" \
+  REDIS_HOST="$host" REDIS_PORT="$(env_value "$SOURCE" FORWARD_REDIS_PORT)" \
+  DEVELOPMENT_QEMU_INSTANCE="$SOURCE" DEVELOPMENT_QEMU_SLOT="$A_SLOT" \
+  DEVELOPMENT_QEMU_DOCKER_NETWORK="$A" DEVELOPMENT_QEMU_COOLIFY_CONTAINER="$A" \
+  "${PRIVILEGED[@]}" sh -c "php artisan dev:qemu $LOCALHOST_PROFILE --as-localhost && php artisan dev:qemu ${TRANSFER_PROFILES[*]}" 2>&1 | grep -E 'Started|rror'
+docker exec "$A" chown -R www-data:www-data /var/www/html/storage/app/ssh
+fetch_vm_key || die 'The VM key (private key 1) is missing on A after seeding.'
+
+echo '== 4. helper image on the VMs'
+helper="$(tinker "$A" 'echo config("constants.coolify.helper_image").":".config("constants.coolify.helper_version");' | tail -1)"
+for profile in "${TRANSFER_PROFILES[@]}"; do
+  # An unreleased helper version cannot be pulled; tag the newest local helper image with that version instead.
+  ssh_vm "$profile" "docker image inspect $helper >/dev/null 2>&1 || docker pull -q $helper >/dev/null 2>&1 || {
+      latest=\$(docker images --format '{{.Repository}}:{{.Tag}}' | grep -E '(^|/)coollabsio/coolify-helper:' | sort -V | tail -1)
+      [ -n \"\$latest\" ] && docker tag \"\$latest\" $helper && echo \"  \$(hostname): tagged \$latest as $helper (not published yet)\"
+    }"
+done
+
+echo '== 5. proxies, API tokens, new code in Horizon'
+tinker "$A" 'foreach (App\Models\Server::where("id", "!=", 0)->get() as $s) { $s->proxy->set("type", "TRAEFIK"); $s->proxy->set("status", "exited"); $s->save(); App\Actions\Proxy\StartProxy::run($s->fresh(), async: false, force: true); echo "  proxy ok: ".$s->name.PHP_EOL; }'
+for side in source target; do
+  C="$A"
+  [[ "$side" == target ]] && C="$B"
+  tinker "$C" '$s = App\Models\InstanceSettings::get(); $s->is_api_enabled = true; $s->save(); session(["currentTeam" => App\Models\Team::find(0)]); echo App\Models\User::find(0)->createToken("transfer-test", ["root"])->plainTextToken;' | tail -1 >"$STATE_DIR/$side.token"
+  chmod 600 "$STATE_DIR/$side.token"
+  docker exec -u www-data "$C" php artisan horizon:terminate >/dev/null
+done
+
+echo '== 6. create and deploy resources on A'
+sleep 10
+"$TT_DIR/seed.sh" >/dev/null || die 'seed.sh failed'
+echo '  waiting for deployments...'
+for _ in $(seq 1 30); do
+  sleep 10
+  [[ "$(tinker "$A" 'echo App\Models\ApplicationDeploymentQueue::whereIn("status", ["queued", "in_progress"])->count();' | tail -1)" == 0 ]] && break
+done
+tinker "$A" 'foreach (App\Models\ApplicationDeploymentQueue::get() as $d) echo "  deploy ".$d->application_name.": ".$d->status.PHP_EOL;'
+for profile in "${TRANSFER_PROFILES[@]}"; do
+  ssh_vm "$profile" 'echo "  $(hostname): $(docker ps --format "{{.Names}}" | grep -v "^coolify-" | wc -l) workload containers"'
+done
+echo "Done. A: $A_URL  B: $B_URL  (login test@example.com / password)"

scripts/transfer-test/reverse-e2e.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Undo paths: the owner marker, transfer back from the archive, a transfer over an archive, take back with automatic
+# release, and deleting an archive.
+# Uses the debian-root VM. Run reset.sh before and after.
+source "$(dirname "$0")/lib.sh"
+TA="$(source_token)"
+TB="$(target_token)"
+PROFILE=debian-root
+S="$(vm_field "$PROFILE" uuid)"
+
+marker() { ssh_vm "$PROFILE" 'cat /data/coolify/transfer/owner.json 2>/dev/null' | jq -r .claim_id; }
+claim_id() { tinker "$1" "echo data_get(App\\Models\\Server::where('uuid','$S')->first()?->server_metadata,'transfer.claim.claim_id');" | tail -1; }
+status() { tinker "$1" "echo data_get(App\\Models\\Server::withTrashed()->where('uuid','$S')->first()?->server_metadata,'transfer.status','none');" | tail -1; }
+# migrate <from port> <from token> <to port> <to token>: transfer through the API, wait, print the transfer status JSON.
+migrate() {
+  local r
+  api POST "$1" "/servers/$S/migrate" "$2" "$(jq -nc --arg u "http://localhost:$3" --arg t "$4" '{target_url:$u,target_token:$t}')" >/dev/null
+  for _ in $(seq 1 40); do sleep 3; r="$(api GET "$1" "/servers/$S/transfer" "$2")"; [[ "$(jq -r .migration.status <<<"$r")" != running ]] && break; done
+  echo "$r"
+}
+
+before="$(stable_workloads "$PROFILE")"
+
+echo '=== 1. A -> B writes the owner marker of B'
+r="$(migrate "$A_PORT" "$TA" "$B_PORT" "$TB")"
+check "transfer A -> B ($(jq -r .migration.status <<<"$r"))" test "$(jq -r .migration.status <<<"$r")" = succeeded
+cb="$(claim_id "$B")"
+check "marker = claim of B ($cb)" test "$(marker)" = "$cb"
+
+echo '=== 2. Transfer back from the Transfer page of the archive on A'
+tinker "$A" "$login_php Livewire\\Livewire::test(App\\Livewire\\Server\\Transfer::class, ['server_uuid'=>'$S'])->set('managerUrl','http://localhost:$B_PORT')->set('managerToken','$TB')->set('thisInstanceUrl','http://localhost:$A_PORT')->call('transferBack');" >/dev/null
+for _ in $(seq 1 40); do sleep 3; [[ "$(status "$A")" == claimed ]] && break; done
+ca="$(claim_id "$A")"
+check 'A claimed (archive replaced), B transferred' test "$(status "$A")/$(status "$B")" = claimed/transferred
+check 'A has one server with this uuid' test "$(tinker "$A" "echo App\\Models\\Server::withTrashed()->where('uuid','$S')->count();" | tail -1)" = 1
+check "marker = claim of A ($ca)" test "$(marker)" = "$ca"
+check 'workloads unchanged after the transfer back' test "$(workloads "$PROFILE")" = "$before"
+
+echo '=== 3. A -> B again replaces the archive on B'
+r="$(migrate "$A_PORT" "$TA" "$B_PORT" "$TB")"
+check "transfer A -> B again ($(jq -r .migration.status <<<"$r"))" test "$(jq -r .migration.status <<<"$r")" = succeeded
+check 'B claimed, A transferred' test "$(status "$B")/$(status "$A")" = claimed/transferred
+check 'workloads unchanged after the second transfer' test "$(workloads "$PROFILE")" = "$before"
+
+echo '=== 4. Take back on A, then B releases the server on its next connection check'
+tinker "$A" "$login_php Livewire\\Livewire::test(App\\Livewire\\Server\\Show::class, ['server_uuid'=>'$S'])->call('enableManagement');" >/dev/null
+ca="$(claim_id "$A")"
+check 'A claimed again' test "$(status "$A")" = claimed
+check "marker = new claim of A ($ca)" test "$(marker)" = "$ca"
+check 'A started the finish list' test "$(tinker "$A" "echo count((array) data_get(App\\Models\\Server::where('uuid','$S')->first()->server_metadata,'transfer.finish.items')) > 0 ? 1 : 0;" | tail -1)" = 1
+check 'B still claimed before its check' test "$(status "$B")" = claimed
+tinker "$B" "(new App\\Jobs\\ServerConnectionCheckJob(App\\Models\\Server::where('uuid','$S')->first()))->handle();" >/dev/null
+check 'B released the server after its check' test "$(status "$B")" = transferred
+check 'B is force disabled' test "$(tinker "$B" "echo (int) App\\Models\\Server::where('uuid','$S')->first()->settings->force_disabled;" | tail -1)" = 1
+tinker "$A" "(new App\\Jobs\\ServerConnectionCheckJob(App\\Models\\Server::where('uuid','$S')->first()))->handle();" >/dev/null
+check 'A keeps the server after its own check' test "$(status "$A")" = claimed
+
+echo '=== 5. Delete the archive on B (records only)'
+tinker "$B" "$login_php Livewire\\Livewire::test(App\\Livewire\\Server\\Delete::class, ['server_uuid'=>'$S'])->call('delete','password');" >/dev/null
+for _ in $(seq 1 20); do sleep 3; [[ "$(status "$B")" == none ]] && break; done
+check 'archive is gone on B' test "$(status "$B")" = none
+check 'workloads unchanged at the end' test "$(workloads "$PROFILE")" = "$before"
+
+echo "=== $FAILS failure(s)"
+exit "$FAILS"

scripts/transfer-test/seed-extras.php

@@ -0,0 +1,92 @@
+<?php
+
+// Fake records on the source (A) around development-qemu-debian-root, for extras-e2e.sh.
+// Fake credentials are enough: the test checks that the records move, not that the providers accept them.
+
+use App\Models\Application;
+use App\Models\DnsProviderZone;
+use App\Models\Environment;
+use App\Models\EnvironmentVariable;
+use App\Models\GithubApp;
+use App\Models\GithubRunnerConfig;
+use App\Models\GithubRunnerExecution;
+use App\Models\IntegrationToken;
+use App\Models\ManagedDnsRecord;
+use App\Models\ResourceLink;
+use App\Models\SecretManagerLink;
+use App\Models\Server;
+use App\Models\SharedEnvironmentVariable;
+use App\Models\StandaloneDocker;
+use App\Models\StandaloneMariadb;
+use App\Services\ProjectIconStorageService;
+
+$quiet = function (string $class, array $attributes) {
+    $model = new $class;
+    $model->forceFill(['uuid' => new_public_id(), ...$attributes])->saveQuietly();
+
+    return $model;
+};
+
+$server = Server::where('uuid', 'development-qemu-debian-root')->firstOrFail();
+$teamId = $server->team_id;
+$app = Application::where('name', 'whoami-2')->firstOrFail();
+$db = StandaloneMariadb::where('name', 'mariadb')->firstOrFail();
+$environment = $app->environment;
+$project = $environment->project;
+
+// Team credentials
+$cloudflare = $quiet(IntegrationToken::class, ['team_id' => $teamId, 'provider' => 'cloudflare', 'name' => 'fake-cloudflare', 'token' => 'fake-cf-token', 'capabilities' => ['dns', 'tunnel']]);
+$infisical = $quiet(IntegrationToken::class, ['team_id' => $teamId, 'provider' => 'infisical', 'name' => 'fake-infisical', 'token' => 'fake-inf-token', 'capabilities' => ['secrets'], 'metadata' => ['client_id' => 'fake-client']]);
+
+// DNS zone + owned record used only by whoami-2
+$zone = $quiet(DnsProviderZone::class, ['integration_token_id' => $cloudflare->id, 'provider_zone_id' => 'fake-zone-id', 'name' => 'example.test', 'account_id' => 'fake-account', 'account_name' => 'Fake']);
+$record = $quiet(ManagedDnsRecord::class, ['team_id' => $teamId, 'integration_token_id' => $cloudflare->id, 'dns_provider_zone_id' => $zone->id, 'provider_record_id' => 'fake-record-id', 'type' => 'A', 'name' => 'whoami2.example.test', 'content' => $server->ip, 'owned' => true]);
+$record->addReference($app);
+
+// Secret manager link
+$quiet(SecretManagerLink::class, ['resourceable_type' => $app->getMorphClass(), 'resourceable_id' => $app->id, 'integration_token_id' => $infisical->id, 'settings' => ['project_id' => 'fake-project', 'environment' => 'prod', 'secret_path' => '/']]);
+
+// Server: Cloudflare tunnel token + custom proxy error page
+$server->settings->forceFill(['cloudflare_tunnel_integration_token_id' => $cloudflare->id, 'cloudflare_tunnel_id' => 'fake-tunnel', 'cloudflare_tunnel_account_id' => 'fake-account'])->saveQuietly();
+$server->forceFill(['proxy_error_page' => '<h1>fake error page</h1>'])->saveQuietly();
+
+// GitHub App + runner config (enabled) + one idle runner
+$githubApp = $quiet(GithubApp::class, ['name' => 'fake-gh-app', 'api_url' => 'https://api.github.com', 'html_url' => 'https://github.com', 'organization' => 'fake-org', 'app_id' => 123, 'installation_id' => 456, 'client_id' => 'fake-client', 'client_secret' => 'fake-secret', 'webhook_secret' => 'fake-webhook', 'is_system_wide' => false, 'is_public' => false, 'team_id' => $teamId, 'private_key_id' => $server->private_key_id]);
+$runnerConfig = $quiet(GithubRunnerConfig::class, ['server_id' => $server->id, 'github_app_id' => $githubApp->id, 'labels' => ['fake-label'], 'is_enabled' => true, 'max_runners' => 2]);
+$quiet(GithubRunnerExecution::class, ['github_app_id' => $githubApp->id, 'github_runner_config_id' => $runnerConfig->id, 'server_id' => $server->id, 'status' => 'idle', 'trigger_workflow_job_id' => 1, 'runner_name' => 'coolify-runner-fake', 'runner_id' => 99, 'labels' => ['fake-label']]);
+
+// A resource of the same project on the OTHER server: same environment (link that stays) and a staging environment
+$ubuntu = Server::where('uuid', 'development-qemu-ubuntu-non-root')->firstOrFail();
+$ubuntuDestination = StandaloneDocker::where('server_id', $ubuntu->id)->firstOrFail();
+$onUbuntu = fn (Environment $env, string $name) => Application::create([
+    'name' => $name, 'environment_id' => $env->id, 'destination_id' => $ubuntuDestination->id, 'destination_type' => $ubuntuDestination->getMorphClass(),
+    'build_pack' => 'dockerimage', 'docker_registry_image_name' => 'traefik/whoami', 'docker_registry_image_tag' => 'latest', 'ports_exposes' => '80',
+    'git_repository' => 'coollabsio/coolify', 'git_branch' => 'main',
+]);
+$otherServerApp = $onUbuntu($environment, 'fake-on-ubuntu');
+$staging = Environment::firstOrCreate(['project_id' => $project->id, 'name' => 'fake-staging']);
+$onUbuntu($staging, 'fake-staging-on-ubuntu');
+
+// Resource links: one between two resources of this server, one to the other server
+$quiet(ResourceLink::class, ['environment_id' => $environment->id, 'source_type' => $app->getMorphClass(), 'source_id' => $app->id, 'target_type' => $db->getMorphClass(), 'target_id' => $db->id, 'label' => 'uses db']);
+$quiet(ResourceLink::class, ['environment_id' => $environment->id, 'source_type' => $app->getMorphClass(), 'source_id' => $app->id, 'target_type' => $otherServerApp->getMorphClass(), 'target_id' => $otherServerApp->id, 'label' => 'calls other']);
+
+// Canvas positions, team shared variable used by an app variable, maintenance mode
+$environment->forceFill(['canvas_positions' => [$app->uuid => ['x' => 111, 'y' => 222], $db->uuid => ['x' => 333, 'y' => 444]]])->saveQuietly();
+SharedEnvironmentVariable::create(['type' => 'team', 'team_id' => $teamId, 'key' => 'FAKE_TEAM_VAR', 'value' => 'team-value']);
+EnvironmentVariable::create(['key' => 'USES_TEAM', 'value' => '{{team.FAKE_TEAM_VAR}}', 'resourceable_type' => $app->getMorphClass(), 'resourceable_id' => $app->id]);
+$app->forceFill(['is_maintenance_enabled' => true, 'maintenance_page' => '<h1>fake maintenance</h1>'])->saveQuietly();
+
+// A git application that uses a system-wide GitHub App, which the transfer does not move
+$systemApp = $quiet(GithubApp::class, ['name' => 'fake-system-wide-app', 'api_url' => 'https://api.github.com', 'html_url' => 'https://github.com', 'team_id' => 0, 'is_system_wide' => true, 'is_public' => false]);
+$gitApp = new Application;
+$gitApp->forceFill([
+    'uuid' => new_public_id(), 'name' => 'git-shop', 'build_pack' => 'nixpacks', 'git_repository' => 'acme/shop', 'git_branch' => 'main', 'ports_exposes' => '3000',
+    'environment_id' => $environment->id, 'destination_id' => $app->destination_id, 'destination_type' => $app->destination_type,
+    'source_type' => GithubApp::class, 'source_id' => $systemApp->id,
+])->saveQuietly();
+
+// Project icon: a 1x1 JPEG
+app(ProjectIconStorageService::class)->storeProjectContents($project, base64_decode('/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////wgALCAABAAEBAREA/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxA='));
+
+echo "seeded\n";

scripts/transfer-test/seed-inventory.php

@@ -1,9 +1,10 @@
 <?php
 
 /**
- * Seed a rich transfer-demo inventory on the current Coolify instance.
- * Run: php artisan tinker scripts/seed-transfer-demo.php
- * Or:  ./scripts/dev exec php artisan tinker --execute "require 'scripts/seed-transfer-demo.php';"
+ * Seed a rich transfer-demo inventory on the current Coolify instance, in the database only: a server
+ * "transfer-full-demo" on the testing-host container with many resource types. Nothing is deployed.
+ * For live transfers between two instances, use the other scripts of scripts/transfer-test (see README.md).
+ * Run: ./scripts/dev exec php artisan tinker --execute "require 'scripts/transfer-test/seed-inventory.php';"
  */
 
 use App\Models\Application;

scripts/transfer-test/seed.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Create real resources on the two transfer VMs of A through the API, then deploy and start them.
+# Run on a fresh A only (reset.sh calls it); it does not clean up. Writes the uuids to <state>/resources.json.
+source "$(dirname "$0")/lib.sh"
+set -e
+
+T="$(source_token)"
+S1="$(vm_field ubuntu-non-root uuid)"
+S2="$(vm_field debian-root uuid)"
+
+call() { # call METHOD PATH [JSON]: fails on an HTTP error
+  local out code
+  out="$(curl -sS -X "$1" "http://127.0.0.1:$A_PORT/api/v1$2" -H "Authorization: Bearer $T" -H 'Content-Type: application/json' -H 'Accept: application/json' ${3:+-d "$3"} -w $'\n%{http_code}')"
+  code="${out##*$'\n'}"
+  out="${out%$'\n'*}"
+  if [[ "$code" -ge 300 ]]; then
+    echo "FAIL $1 $2 ($code): $out" >&2
+    return 1
+  fi
+  echo "$out"
+}
+uuid() { jq -r '.uuid'; }
+
+echo '== projects'
+P1=$(call POST /projects '{"name":"Transfer Demo","description":"Resources on the ubuntu non-root VM"}' | uuid)
+P2=$(call POST /projects '{"name":"Second Server","description":"Resources on the debian root VM"}' | uuid)
+call POST "/projects/$P1/environments" '{"name":"staging"}' >/dev/null
+
+echo '== server 1 (ubuntu non-root): Docker image app with variables, volume, tag, scheduled task'
+APP1=$(call POST /applications/dockerimage "$(jq -nc --arg p "$P1" --arg s "$S1" '{project_uuid:$p,server_uuid:$s,environment_name:"production",name:"whoami",docker_registry_image_name:"traefik/whoami",docker_registry_image_tag:"latest",ports_exposes:"80",instant_deploy:false}')" | uuid)
+call POST "/applications/$APP1/envs" '{"key":"WHOAMI_NAME","value":"transfer-demo"}' >/dev/null
+call POST "/applications/$APP1/envs" '{"key":"APP_SECRET","value":"very-secret-value"}' >/dev/null
+call POST "/applications/$APP1/storages" '{"type":"persistent","name":"whoami-data","mount_path":"/data"}' >/dev/null || true
+call POST "/applications/$APP1/tags" '{"tag_name":"transfer-demo"}' >/dev/null || true
+call POST "/applications/$APP1/scheduled-tasks" '{"name":"hello","command":"echo hello","frequency":"*/30 * * * *"}' >/dev/null || true
+
+echo '== server 1: Dockerfile app (staging)'
+DF=$(printf 'FROM nginx:alpine\nRUN echo "built by Coolify A" > /usr/share/nginx/html/index.html\nEXPOSE 80\n' | base64 -w0)
+APP2=$(call POST /applications/dockerfile "$(jq -nc --arg p "$P1" --arg s "$S1" --arg d "$DF" '{project_uuid:$p,server_uuid:$s,environment_name:"staging",name:"nginx-dockerfile",dockerfile:$d,ports_exposes:"80",instant_deploy:false}')" | uuid)
+
+echo '== server 1: PostgreSQL with a backup, Redis'
+PG=$(call POST /databases/postgresql "$(jq -nc --arg p "$P1" --arg s "$S1" '{project_uuid:$p,server_uuid:$s,environment_name:"production",name:"postgres",postgres_user:"demo",postgres_password:"pg-secret-password",postgres_db:"demo",instant_deploy:false}')" | uuid)
+call POST "/databases/$PG/backups" '{"frequency":"0 3 * * *","enabled":true,"save_s3":false,"databases_to_backup":"demo"}' >/dev/null || true
+RD=$(call POST /databases/redis "$(jq -nc --arg p "$P1" --arg s "$S1" '{project_uuid:$p,server_uuid:$s,environment_name:"production",name:"redis",instant_deploy:false}')" | uuid)
+
+echo '== server 1: one-click service (uptime-kuma)'
+SV=$(call POST /services "$(jq -nc --arg p "$P1" --arg s "$S1" '{type:"uptime-kuma",project_uuid:$p,server_uuid:$s,environment_name:"production",name:"uptime-kuma",instant_deploy:false}')" | uuid)
+
+echo '== server 2 (debian root): whoami-2, MariaDB'
+APP3=$(call POST /applications/dockerimage "$(jq -nc --arg p "$P2" --arg s "$S2" '{project_uuid:$p,server_uuid:$s,environment_name:"production",name:"whoami-2",docker_registry_image_name:"traefik/whoami",docker_registry_image_tag:"latest",ports_exposes:"80",instant_deploy:false}')" | uuid)
+MD=$(call POST /databases/mariadb "$(jq -nc --arg p "$P2" --arg s "$S2" '{project_uuid:$p,server_uuid:$s,environment_name:"production",name:"mariadb",instant_deploy:false}')" | uuid)
+
+echo '== deploy and start everything'
+for u in $APP1 $APP2 $APP3; do call POST "/deploy?uuid=$u" >/dev/null && echo "deploy queued $u"; done
+for u in $PG $RD $MD; do call POST "/databases/$u/start" >/dev/null && echo "start queued $u"; done
+call POST "/services/$SV/start" >/dev/null && echo "start queued $SV"
+
+jq -n --arg p1 "$P1" --arg p2 "$P2" --arg app1 "$APP1" --arg app2 "$APP2" --arg app3 "$APP3" --arg pg "$PG" --arg rd "$RD" --arg md "$MD" --arg sv "$SV" \
+  '{projects:[$p1,$p2],apps:[$app1,$app2,$app3],databases:[$pg,$rd,$md],service:$sv}' | tee "$STATE_DIR/resources.json"

scripts/transfer-test/setup.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Prepare the transfer test kit: start the target instance (B) next to the source (A), and let them reach
+# each other and A's VMs. Safe to run again, for example after a host reboot. Linux with KVM only.
+#
+# Usage: scripts/transfer-test/setup.sh    (then scripts/transfer-test/reset.sh)
+source "$(dirname "$0")/lib.sh"
+
+[[ -c /dev/kvm ]] || die 'The kit needs KVM VMs (Linux). See scripts/transfer-test/README.md.'
+running "$A" || die "Start the source instance first: ./scripts/dev start"
+
+echo "== 1. target instance $TARGET (side by side, testing-host as its localhost)"
+# On a new instance, its server seeding can run before the migrations. That does not matter: reset.sh seeds B again.
+COOLIFY_DEV_INSTANCE="$TARGET" COOLIFY_DEV_SIDE_BY_SIDE=1 COOLIFY_DEV_SERVER_BACKEND=testing-host ./scripts/dev start ||
+  echo '  (the seeding of the new instance failed; reset.sh seeds both databases again)'
+running "$B" || die "The target instance $B does not run."
+target_values
+
+echo "== 2. firewall: B may reach the VM network of A (10.221.$A_SLOT.0/24)"
+subnet="$(docker network inspect "$B" --format '{{(index .IPAM.Config 0).Subnet}}')"
+rule=(LIBVIRT_FWI -s "$subnet" -d "10.221.$A_SLOT.0/24" -o "cdevbr$A_SLOT" -j ACCEPT)
+if "${PRIVILEGED[@]}" iptables -C "${rule[@]}" 2>/dev/null; then
+  echo "  rule exists"
+else
+  "${PRIVILEGED[@]}" iptables -I "${rule[0]}" 1 "${rule[@]:1}" && echo "  rule added for $subnet"
+fi
+
+echo "== 3. A and B reach each other's app port"
+bind="$(grep -E '^DEV_BIND_ADDRESS=' .env 2>/dev/null | tail -1 | cut -d= -f2-)"
+if [[ "${bind:-0.0.0.0}" == 127.0.0.1 ]]; then
+  # The ports listen on 127.0.0.1 only; containers reach the host through the docker0 gateway.
+  gateway="$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}')"
+  for port in "$A_PORT" "$B_PORT"; do
+    name="coolify-transfer-test-forward-$port"
+    if running "$name"; then
+      echo "  $name runs"
+    else
+      docker rm -f "$name" >/dev/null 2>&1 || true
+      docker run -d --name "$name" --network host --restart unless-stopped alpine/socat \
+        "tcp-listen:$port,bind=$gateway,fork,reuseaddr" "tcp:127.0.0.1:$port" >/dev/null && echo "  $name started ($gateway:$port -> 127.0.0.1:$port)"
+    fi
+  done
+else
+  echo "  not needed: the ports listen on ${bind:-0.0.0.0}"
+fi
+
+echo "Done. A: $A_URL  B: $B_URL"
+echo "Next: scripts/transfer-test/reset.sh"

scripts/transfer-test/verify-extras-source.php

@@ -0,0 +1,28 @@
+<?php
+
+// Checks on the source (A) after the debian-root transfer of extras-e2e.sh. Prints PASS/FAIL lines.
+
+use App\Models\GithubRunnerConfig;
+use App\Models\GithubRunnerExecution;
+use App\Models\ManagedDnsRecord;
+use App\Models\Server;
+
+$check = function (string $name, bool $ok, string $detail = '') {
+    echo ($ok ? 'PASS ' : 'FAIL ').$name.($detail !== '' ? " ({$detail})" : '')."\n";
+};
+
+$server = Server::where('uuid', 'development-qemu-debian-root')->first();
+$migration = (array) data_get($server?->server_metadata, 'migration');
+$messages = collect($migration['log'] ?? [])->pluck('message');
+$check('source is transferred away', $server?->isTransferredAway() === true);
+$check('transfer succeeded', ($migration['status'] ?? null) === 'succeeded', (string) ($migration['error'] ?? ''));
+$check('precheck NOTE about resources on the other server', $messages->contains(fn ($m) => str_starts_with($m, "NOTE: project 'Second Server' also has 2 resources on other servers")));
+$check('runner config disabled on the source', GithubRunnerConfig::where('server_id', $server?->id)->value('is_enabled') === false);
+$check('runner execution cancelled on the source', GithubRunnerExecution::where('runner_name', 'coolify-runner-fake')->value('status')?->value === 'cancelled');
+$record = ManagedDnsRecord::where('name', 'whoami2.example.test')->first();
+$check('source keeps the DNS record for a take back, not owned', $record !== null && $record->owned === false);
+$check('precheck NOTE about the system-wide GitHub App', $messages->contains(fn ($m) => str_starts_with($m, "NOTE: the system-wide GitHub App 'fake-system-wide-app'")));
+$checklist = implode("\n", (array) ($migration['checklist'] ?? []));
+$check('checklist: GitHub App URLs', str_contains($checklist, 'GitHub App fake-gh-app'));
+$check('checklist: enable GitHub runners', str_contains($checklist, 'enable GitHub runners'));
+echo "--- warnings\n".implode("\n", (array) ($migration['warnings'] ?? []))."\n--- checklist\n{$checklist}\n";

scripts/transfer-test/verify-extras-target.php

@@ -0,0 +1,69 @@
+<?php
+
+// Checks on the target (B) after the debian-root transfer of extras-e2e.sh. Prints PASS/FAIL lines.
+
+use App\Models\Application;
+use App\Models\Environment;
+use App\Models\GithubApp;
+use App\Models\GithubRunnerConfig;
+use App\Models\IntegrationToken;
+use App\Models\ManagedDnsRecord;
+use App\Models\ResourceLink;
+use App\Models\SecretManagerLink;
+use App\Models\Server;
+use App\Models\SharedEnvironmentVariable;
+use App\Models\StandaloneMariadb;
+use App\Services\ProjectIconStorageService;
+
+$check = function (string $name, bool $ok, string $detail = '') {
+    echo ($ok ? 'PASS ' : 'FAIL ').$name.($detail !== '' ? " ({$detail})" : '')."\n";
+};
+
+$server = Server::where('uuid', 'development-qemu-debian-root')->first();
+$check('server is on B and claimed', data_get($server?->server_metadata, 'transfer.status') === 'claimed');
+$check('custom proxy error page', $server?->proxy_error_page === '<h1>fake error page</h1>');
+
+$cloudflare = IntegrationToken::where('name', 'fake-cloudflare')->first();
+$infisical = IntegrationToken::where('name', 'fake-infisical')->first();
+$check('Cloudflare token moved with its secret', $cloudflare?->token === 'fake-cf-token');
+$check('Infisical token moved with its secret', $infisical?->token === 'fake-inf-token');
+$check('Cloudflare tunnel uses the moved token', $server?->settings?->cloudflare_tunnel_integration_token_id === $cloudflare?->id && $server?->settings?->cloudflare_tunnel_id === 'fake-tunnel');
+
+$app = Application::where('name', 'whoami-2')->first();
+$link = SecretManagerLink::where('resourceable_type', $app?->getMorphClass())->where('resourceable_id', $app?->id)->first();
+$check('secret manager link moved to the moved token', $link?->integration_token_id === $infisical?->id && data_get($link?->settings, 'project_id') === 'fake-project');
+
+$record = ManagedDnsRecord::where('name', 'whoami2.example.test')->first();
+$check('DNS record + zone moved', $record !== null && $record->zone?->provider_zone_id === 'fake-zone-id' && $record->integration_token_id === $cloudflare?->id);
+$check('DNS record references the moved app', $record?->references()->where('resource_type', $app?->getMorphClass())->where('resource_id', $app?->id)->exists() === true);
+// Same app.id on both instances: the owner comment does not change, so the record stays owned.
+// Different app.id: the target must change the comment at Cloudflare, which fails with a fake token.
+$sourceAppId = trim((string) @file_get_contents('/tmp/source-app-id'));
+$sameInstanceId = $sourceAppId === (string) config('app.id');
+$check('DNS ownership '.($sameInstanceId ? 'kept (same instance ID, no adoption needed)' : 'dropped (fake token cannot adopt)'), $record?->owned === $sameInstanceId);
+
+$githubApp = GithubApp::where('name', 'fake-gh-app')->first();
+$runner = GithubRunnerConfig::where('server_id', $server?->id)->first();
+$check('GitHub App moved (only the runner config uses it)', $githubApp !== null && $githubApp->organization === 'fake-org');
+$check('runner config moved, DISABLED, same App and labels', $runner !== null && $runner->is_enabled === false && $runner->github_app_id === $githubApp?->id && in_array('fake-label', (array) $runner->labels, true));
+
+$db = StandaloneMariadb::where('name', 'mariadb')->first();
+$check('resource link between the two moved resources', ResourceLink::where('source_id', $app?->id)->where('target_type', $db?->getMorphClass())->where('target_id', $db?->id)->where('label', 'uses db')->exists());
+$check('resource link to the other server is not moved', ! ResourceLink::where('label', 'calls other')->exists());
+
+$environment = $app?->environment;
+$check('canvas positions moved', data_get($environment?->canvas_positions, ($app?->uuid).'.x') === 111);
+$check('environment without resources on this server is not moved', ! Environment::where('name', 'fake-staging')->exists());
+$check('apps on the other server are not moved', ! Application::whereIn('name', ['fake-on-ubuntu', 'fake-staging-on-ubuntu'])->exists());
+
+$check('team shared variable created', SharedEnvironmentVariable::where('type', 'team')->where('key', 'FAKE_TEAM_VAR')->value('value') === 'team-value');
+$check('app variable that uses it moved', $app?->environment_variables()->where('key', 'USES_TEAM')->value('value') === '{{team.FAKE_TEAM_VAR}}');
+$check('maintenance mode moved', $app?->is_maintenance_enabled === true && $app?->maintenance_page === '<h1>fake maintenance</h1>');
+
+$gitShop = Application::where('name', 'git-shop')->first();
+$check('git-shop moved without its system-wide source', $gitShop !== null && $gitShop->source_id === null);
+$check('finish list has a git source step for git-shop', collect(data_get($server?->server_metadata, 'transfer.finish.items'))->contains('key', 'git_source:'.$gitShop?->uuid));
+
+$project = $environment?->project;
+$icon = $project ? app(ProjectIconStorageService::class)->projectContents($project) : null;
+$check('project icon moved', $icon !== null && str_starts_with($icon, "\xFF\xD8\xFF"));