coolify/scripts/transfer-test/e2e.sh

#!/usr/bin/env bash
# Transfer both VM servers A -> B through the API and check source, target, server, and a redeploy from B.
# Run reset.sh before and after.
source "$(dirname "$0")/lib.sh"
TA="$(source_token)"
TB="$(target_token)"

for profile in "${TRANSFER_PROFILES[@]}"; do
  S="$(vm_field "$profile" uuid)"
  ip="$(vm_field "$profile" ip)"
  echo "=== $S"
  before="$(stable_workloads "$profile")"
  apps="$(tinker "$A" "echo App\\Models\\Server::where('uuid','$S')->first()->applications()->pluck('uuid')->implode(' ');" | tail -1)"

  echo '-- start the transfer on A'
  r="$(api POST "$A_PORT" "/servers/$S/migrate" "$TA" "$(jq -nc --arg u "http://localhost:$B_PORT" --arg t "$TB" '{target_url:$u,target_token:$t}')")"
  if [[ "$(jq -r .migration.status <<<"$r" 2>/dev/null)" != running ]]; then
    bad "transfer request accepted: $r"
    continue
  fi
  ok 'transfer request accepted'
  for _ in $(seq 1 40); do sleep 3; st="$(api GET "$A_PORT" "/servers/$S/transfer" "$TA" | jq -r .migration.status)"; [[ "$st" != running ]] && break; done
  check "transfer finished ($st)" test "$st" = succeeded

  echo '-- source A'
  a="$(api GET "$A_PORT" "/servers/$S/transfer" "$TA")"
  export_id="$(jq -r .transfer.export_id <<<"$a")"
  check 'A transfer.status = transferred' test "$(jq -r .transfer.status <<<"$a")" = transferred
  check 'A force_disabled + Sentinel off' test "$(tinker "$A" "\$s=App\\Models\\Server::where('uuid','$S')->first()->settings; echo (int)\$s->force_disabled.(int)\$s->is_sentinel_enabled;" | tail -1)" = 10
  check 'A shows the take-back modal' test "$(tinker "$A" "$login_php echo (int) str_contains(Livewire\\Livewire::test(App\\Livewire\\Server\\Show::class, ['server_uuid'=>'$S'])->html(), 'Take back a transferred server?');" | tail -1)" = 1
  r="$(api POST "$A_PORT" "/servers/$S/migrate" "$TA" "$(jq -nc --arg u "http://localhost:$B_PORT" --arg t "$TB" '{target_url:$u,target_token:$t}')")"
  check 'second transfer refused by the prechecks' grep -q 'already transferred' <<<"$r"
  r="$(api DELETE "$A_PORT" "/servers/$S?delete_from_provider=true" "$TA")"
  check 'A refuses to delete the cloud server of the archive' grep -q 'managed by another Coolify instance' <<<"$r"

  echo '-- target B'
  b="$(api GET "$B_PORT" "/servers/$S/transfer" "$TB")"
  check 'B transfer.status = claimed' test "$(jq -r .transfer.status <<<"$b")" = claimed
  check 'same export id on A and B' test "$(jq -r .transfer.export_id <<<"$b")" = "$export_id"
  for _ in $(seq 1 20); do
    sleep 3
    meta="$(tinker "$B" "\$s=App\\Models\\Server::where('uuid','$S')->first(); echo json_encode(['os'=>data_get(\$s->server_metadata,'os'),'reach'=>\$s->settings->is_reachable,'usable'=>\$s->settings->is_usable,'logs'=>\$s->validation_logs,'transfer'=>data_get(\$s->server_metadata,'transfer.status')]);" | tail -1)"
    [[ "$(jq -r .os <<<"$meta")" != null ]] && break
  done
  check "automatic validation filled the metadata ($(jq -r .os <<<"$meta"))" test "$(jq -r .os <<<"$meta")" != null
  check 'B reachable + usable, no validation error' test "$(jq -c '[.reach,.usable,.logs]' <<<"$meta")" = '[true,true,null]'
  check 'transfer state kept after validation' test "$(jq -r .transfer <<<"$meta")" = claimed
  check 'B started the finish list' test "$(tinker "$B" "echo count((array) data_get(App\\Models\\Server::where('uuid','$S')->first()->server_metadata,'transfer.finish.items')) > 0 ? 1 : 0;" | tail -1)" = 1
  check 'A links to the server on B' test "$(jq -r .migration.target_server_url <<<"$a")" = "$B_URL/server/$S"
  check 'B has all applications' test "$(tinker "$B" "echo App\\Models\\Server::where('uuid','$S')->first()->applications()->pluck('uuid')->implode(' ');" | tail -1)" = "$apps"
  check 'B validates the server over SSH' test "$(tinker "$B" "\$r=App\\Models\\Server::where('uuid','$S')->first()->validateConnection(); echo (int)(bool)\$r['uptime'];" | tail -1)" = 1

  echo '-- server'
  sleep 5
  after="$(workloads "$profile")"
  check 'proxy + workloads not recreated' test "$before" = "$after"
  sentinel_env="$(ssh_vm "$profile" "docker inspect coolify-sentinel --format '{{range .Config.Env}}{{println .}}{{end}}'")"
  check 'Sentinel sends to B' grep -qx "PUSH_ENDPOINT=$B_URL" <<<"$sentinel_env"
  for app in $apps; do check "app $app answers over HTTP" bash -c "curl -sf -m5 -H 'Host: $app.$ip.sslip.io' http://$ip/ >/dev/null"; done

  echo '-- redeploy one app from B'
  app="${apps%% *}"
  api POST "$B_PORT" "/deploy?uuid=$app&force=false" "$TB" >/dev/null
  for _ in $(seq 1 40); do sleep 5; ds="$(tinker "$B" "echo App\\Models\\ApplicationDeploymentQueue::where('application_id', App\\Models\\Application::where('uuid','$app')->value('id'))->latest('id')->value('status');" | tail -1)"; [[ "$ds" == finished || "$ds" == failed ]] && break; done
  check "redeploy from B ($ds)" test "$ds" = finished
  check 'app answers after the redeploy' bash -c "for i in {1..10}; do curl -sf -m5 -H 'Host: $app.$ip.sslip.io' http://$ip/ >/dev/null && exit 0; sleep 3; done; exit 1"

  [[ "$profile" == ubuntu-non-root ]] || continue
  echo '-- operations from B over the non-root SSH user'
  uuid_on_b() { tinker "$B" "echo App\\Models\\$1::where('name','$2')->value('uuid');" | tail -1; }
  pg="$(uuid_on_b StandalonePostgresql postgres)"
  rd="$(uuid_on_b StandaloneRedis redis)"
  sv="$(uuid_on_b Service uptime-kuma)"
  nginx="$(uuid_on_b Application nginx-dockerfile)"
  for u in "$pg" "$rd"; do api POST "$B_PORT" "/databases/$u/restart" "$TB" >/dev/null; done
  api POST "$B_PORT" "/services/$sv/restart" "$TB" >/dev/null
  sleep 20
  stable_workloads "$profile" >/dev/null
  running_after="$(workloads "$profile")"
  for u in "$pg" "$rd" "$sv"; do check "restarted from B: $u runs" grep -q "$u" <<<"$running_after"; done
  tinker "$B" "App\\Jobs\\DatabaseBackupJob::dispatchSync(App\\Models\\StandalonePostgresql::where('uuid','$pg')->first()->scheduledBackups()->firstOrFail());" >/dev/null
  r="$(tinker "$B" "echo App\\Models\\StandalonePostgresql::where('uuid','$pg')->first()->scheduledBackups()->first()->executions()->latest('id')->value('status');" | tail -1)"
  check "database backup from B ($r)" test "$r" = success
  task="$(api POST "$B_PORT" "/applications/$nginx/scheduled-tasks" "$TB" '{"name":"from-b","command":"echo from-b","frequency":"0 0 * * *"}' | jq -r .uuid)"
  api POST "$B_PORT" "/applications/$nginx/scheduled-tasks/$task/execute" "$TB" >/dev/null
  for _ in $(seq 1 20); do
    sleep 3
    r="$(api GET "$B_PORT" "/applications/$nginx/scheduled-tasks/$task/executions" "$TB" | jq -r '.[0].status // empty')"
    [[ "$r" == success || "$r" == failed ]] && break
  done
  check "scheduled task from B ($r)" test "$r" = success
done

echo "=== $FAILS failure(s)"
exit "$FAILS"