memind/scripts/release-portal-canary-prod.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HOST="${STUDIO_HOST:-58.38.22.103}"
REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}"
STABLE_DIR="${REMOTE_ROOT}/Memind"
CANDIDATE_BASE="${REMOTE_ROOT}/Memind-candidates"
INCOMING_DIR="${REMOTE_ROOT}/incoming/memind-portal-canary"
BACKUP_DIR="${REMOTE_ROOT}/backups/memind"
GOOSED_DIR="${REMOTE_ROOT}/goosed-prod"
RUNTIME_ROOT="${ROOT}/.runtime/portal"
PORTAL_CANDIDATE_LABEL="cn.tkmind.memind-portal-candidate"
CANARY_PROXY_LABEL="cn.tkmind.memind-canary-proxy"
CANARY_TUNNEL_LABEL="cn.tkmind.memind-canary-tunnel"
DEEPSEEK_COMPAT_LABEL="cn.tkmind.memind-deepseek-compat-candidate"
EDGE_HOST="${MEMIND_CANARY_EDGE_HOST:-ssh105-public}"
CANARY_PROXY_PORT=18082
CANARY_TUNNEL_REMOTE_PORT=19082
DEEPSEEK_COMPAT_PORT=18036
EDGE_MOBILE_CONFIG="/etc/nginx/conf.d/m.tkmind.cn.conf"
EDGE_WECHAT_CONFIG="/etc/nginx/conf.d/wechat.m.tkmind.cn.conf"
CANARY_USERNAMES="john"
CANARY_WECHAT_USER_IDS="wx_ul610et8"
RELEASE_TS="$(date +%Y%m%d-%H%M%S)"
FULL_SHA="$(git -C "${ROOT}" rev-parse HEAD)"
SHORT_SHA="${FULL_SHA:0:7}"
RELEASE_ID="${RELEASE_TS}-${SHORT_SHA}"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/memind-portal-canary-release.XXXXXX")"
BUNDLE_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.tar.gz"
MANIFEST_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.manifest.txt"
SHA_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.sha256"
DRY_RUN=0
AUTO_YES=0

cleanup() {
  rm -rf "${TMP_DIR}"
}
trap cleanup EXIT

usage() {
  cat <<'EOF'
Usage:
  bash scripts/release-portal-canary-prod.sh [--dry-run] [--yes]

Deploys a candidate Portal beside the stable 103 runtime:
  stable Portal     127.0.0.1:8081
  canary router     127.0.0.1:18082
  candidate Portal  127.0.0.1:18081
  candidate goosed  127.0.0.1:18015
  DeepSeek compat   0.0.0.0:18036 (host/container only)

The stable runtime is not replaced. A dedicated reverse tunnel exposes the
router only to 105 at 127.0.0.1:19082. The committed release workflow updates
the two active 105 nginx upstreams only after backups, candidate health, and
immutable identity checks pass, then proves automatic fallback before success.
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run) DRY_RUN=1 ;;
    --yes|-y) AUTO_YES=1 ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
  shift
done

for bypass in \
  ALLOW_PORTAL_RELEASE_SCOPE_BYPASS \
  ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES \
  MEMIND_RELEASE_SKIP_GATE; do
  if [[ "${!bypass:-0}" == "1" ]]; then
    echo "Production canary release forbids ${bypass}." >&2
    exit 1
  fi
done

say() {
  printf '\n[%s] %s\n' "$(date +%H:%M:%S)" "$*"
}

need_cmd() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "Missing command: $1" >&2
    exit 1
  }
}

need_cmd ssh
need_cmd scp
need_cmd tar
need_cmd shasum

branch="$(git -C "${ROOT}" branch --show-current)"
if [[ "${branch}" != "main" ]]; then
  echo "Canary production release requires main, got: ${branch:-detached}" >&2
  exit 1
fi
git -C "${ROOT}" fetch origin --prune
if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "$(git -C "${ROOT}" rev-parse origin/main)" ]]; then
  echo "Canary production release requires HEAD to equal origin/main." >&2
  exit 1
fi
ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh" --skip-fetch

say "Run release source guards"
(
  cd "${ROOT}"
  npm test -- --test-name-pattern='publish|space|billing' >/dev/null
  npm run verify:mindspace-publish-guards >/dev/null
  npm run verify:mindspace-page-sync-guards >/dev/null
  npm run verify:h5-session-patches >/dev/null
  npm run verify:page-data >/dev/null
  npm run check:mindspace-public-links >/dev/null
)

required_runtime_paths=(
  server.mjs
  memind-canary-proxy.mjs
  deepseek-no-think-proxy.mjs
  wechat-mp.bundle.mjs
  mindspace-sandbox-mcp.mjs
  tkmind-search-mcp.mjs
  tkmind-excel-mcp.mjs
  dist
  package.json
  scripts/run-memind-portal-candidate.sh
  scripts/run-memind-canary-proxy-prod.sh
  scripts/run-deepseek-compat-proxy-candidate.sh
  scripts/goosed-canary.compose.yml
  scripts/memind-portal-tunnel.sh
)
for relative in "${required_runtime_paths[@]}"; do
  [[ -e "${RUNTIME_ROOT}/${relative}" ]] || {
    echo "Candidate runtime is missing ${relative}; rebuild and rerun the complete Gate." >&2
    exit 1
  }
done

say "Verify the complete Gate report against the exact candidate artifact"
node "${ROOT}/scripts/verify-release-gate-report.mjs" --artifact "${RUNTIME_ROOT}"

say "Package the verified candidate artifact"
tar -czf "${BUNDLE_PATH}" -C "${RUNTIME_ROOT}" .
bundle_sha="$(shasum -a 256 "${BUNDLE_PATH}" | awk '{print $1}')"
printf '%s  %s\n' "${bundle_sha}" "$(basename "${BUNDLE_PATH}")" > "${SHA_PATH}"
{
  echo "release_id=${RELEASE_ID}"
  echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')"
  echo "git_head=${FULL_SHA}"
  echo "git_branch=${branch}"
  echo "artifact_tree=.runtime/portal"
  echo "artifact_bundle_sha256=${bundle_sha}"
  echo "routing=stable:8081,proxy:18082,edge-tunnel:19082,candidate:18081,goosed-canary:18015,deepseek-compat:18036"
  echo "canary_usernames=${CANARY_USERNAMES}"
  echo "canary_wechat_user_ids=${CANARY_WECHAT_USER_IDS}"
  echo "manual_environment_changes=none"
} > "${MANIFEST_PATH}"

if [[ "${DRY_RUN}" -eq 1 ]]; then
  say "Dry-run complete; no 103 connection or production write occurred"
  ls -lh "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}"
  exit 0
fi

say "Run 103 read-only preflight"
ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" \
  "test -d '${STABLE_DIR}' \
   && test -f '${STABLE_DIR}/.env' \
   && test -f '${GOOSED_DIR}/docker-compose.prod.yml' \
   && curl -fsS http://127.0.0.1:8081/api/status >/dev/null \
   && ! lsof -nP -iTCP:${CANARY_PROXY_PORT} -sTCP:LISTEN >/dev/null 2>&1 \
   && ! lsof -nP -iTCP:${DEEPSEEK_COMPAT_PORT} -sTCP:LISTEN >/dev/null 2>&1 \
   && ! lsof -nP -iTCP:18081 -sTCP:LISTEN >/dev/null 2>&1 \
   && ! lsof -nP -iTCP:18015 -sTCP:LISTEN >/dev/null 2>&1 \
   && /opt/homebrew/bin/docker inspect goosed-prod-1 >/dev/null \
   && test \"\$(df -Pk '${REMOTE_ROOT}' | awk 'NR==2 {print \$4}')\" -gt 10485760 \
   && ssh -o BatchMode=yes -o ConnectTimeout=10 '${EDGE_HOST}' \
      \"test -f '${EDGE_MOBILE_CONFIG}' \
       && test -f '${EDGE_WECHAT_CONFIG}' \
       && grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}' \
       && grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}' \
       && ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}' \
       && ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}' \
       && nginx -t >/dev/null 2>&1 \
       && ! lsof -nP -iTCP:${CANARY_TUNNEL_REMOTE_PORT} -sTCP:LISTEN >/dev/null 2>&1\""

if [[ "${AUTO_YES}" -ne 1 ]]; then
  say "Production canary confirmation"
  echo "Target host: ${HOST}"
  echo "Release: ${RELEASE_ID}"
  echo "Commit: ${FULL_SHA}"
  echo "Stable remains on 8081; 105 will use the isolated 19082 canary tunnel."
  echo "Selected immutable identities route to candidate 18081."
  read -r -p "Continue with the 103 canary release? [y/N] " confirm </dev/tty
  [[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
fi

say "Upload the verified candidate bundle"
ssh -o BatchMode=yes "${HOST}" "mkdir -p '${INCOMING_DIR}' '${BACKUP_DIR}' '${CANDIDATE_BASE}'"
scp -q "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}" "${HOST}:${INCOMING_DIR}/"

say "Install the isolated candidate and identity router on 103"
ssh -o BatchMode=yes "${HOST}" \
  "RELEASE_ID='${RELEASE_ID}' \
   STABLE_DIR='${STABLE_DIR}' \
   CANDIDATE_BASE='${CANDIDATE_BASE}' \
   INCOMING_DIR='${INCOMING_DIR}' \
   BACKUP_DIR='${BACKUP_DIR}' \
   GOOSED_DIR='${GOOSED_DIR}' \
   PORTAL_CANDIDATE_LABEL='${PORTAL_CANDIDATE_LABEL}' \
   CANARY_PROXY_LABEL='${CANARY_PROXY_LABEL}' \
   CANARY_TUNNEL_LABEL='${CANARY_TUNNEL_LABEL}' \
   DEEPSEEK_COMPAT_LABEL='${DEEPSEEK_COMPAT_LABEL}' \
   EDGE_HOST='${EDGE_HOST}' \
   CANARY_PROXY_PORT='${CANARY_PROXY_PORT}' \
   CANARY_TUNNEL_REMOTE_PORT='${CANARY_TUNNEL_REMOTE_PORT}' \
   DEEPSEEK_COMPAT_PORT='${DEEPSEEK_COMPAT_PORT}' \
   EDGE_MOBILE_CONFIG='${EDGE_MOBILE_CONFIG}' \
   EDGE_WECHAT_CONFIG='${EDGE_WECHAT_CONFIG}' \
   CANARY_USERNAMES='${CANARY_USERNAMES}' \
   CANARY_WECHAT_USER_IDS='${CANARY_WECHAT_USER_IDS}' \
   /bin/bash" <<'REMOTE_SCRIPT'
set -euo pipefail

LAUNCHD_GUI="gui/$(id -u)"
CANDIDATE_DIR="${CANDIDATE_BASE}/${RELEASE_ID}"
BUNDLE="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.tar.gz"
MANIFEST="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.manifest.txt"
SHA_FILE="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.sha256"
FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before-canary.tar.gz"
PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before-canary.tar.gz"
CANDIDATE_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_CANDIDATE_LABEL}.plist"
PROXY_PLIST="${HOME}/Library/LaunchAgents/${CANARY_PROXY_LABEL}.plist"
CANARY_TUNNEL_PLIST="${HOME}/Library/LaunchAgents/${CANARY_TUNNEL_LABEL}.plist"
DEEPSEEK_COMPAT_PLIST="${HOME}/Library/LaunchAgents/${DEEPSEEK_COMPAT_LABEL}.plist"
SECRET_FILE="${HOME}/.config/memind/canary-router.secret"
DOCKER_BIN="/opt/homebrew/bin/docker"
GOOSED_COMPOSE="${GOOSED_DIR}/docker-compose.prod.yml"
EDGE_MOBILE_BACKUP="${EDGE_MOBILE_CONFIG}.before-canary-${RELEASE_ID}"
EDGE_WECHAT_BACKUP="${EDGE_WECHAT_CONFIG}.before-canary-${RELEASE_ID}"

say() {
  printf '\n[remote %s] %s\n' "$(date +%H:%M:%S)" "$*"
}

candidate_healthy() {
  local headers code
  headers="$(mktemp)"
  code="$(
    curl -sS -D "${headers}" -o /dev/null -w '%{http_code}' \
      http://127.0.0.1:18081/api/status 2>/dev/null || true
  )"
  if [[ "${code}" == "200" ]] \
    && grep -qi '^x-memind-runtime-role: candidate' "${headers}"; then
    rm -f "${headers}"
    return 0
  fi
  rm -f "${headers}"
  return 1
}

deepseek_compat_healthy() {
  curl -fsS --max-time 5 "http://127.0.0.1:${DEEPSEEK_COMPAT_PORT}/health" \
    | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
        let input = "";
        process.stdin.on("data", (chunk) => { input += chunk; });
        process.stdin.on("end", () => {
          const status = JSON.parse(input);
          if (
            status.ok !== true
            || status.contractVersion !== 1
            || status.deepseekThinking !== "disabled"
          ) process.exit(1);
        });
      '
}

edge_ssh() {
  # This remote release script itself arrives on stdin. Ordinary nested SSH
  # commands must not consume the unparsed remainder of that script.
  ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${EDGE_HOST}" "$@"
}

write_canary_tunnel_plist() {
  cat > "${CANARY_TUNNEL_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${CANARY_TUNNEL_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${STABLE_DIR}/scripts/memind-portal-tunnel.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_PORTAL_TUNNEL_HOST</key><string>${EDGE_HOST}</string>
    <key>MEMIND_PORTAL_TUNNEL_LOCAL_PORT</key><string>${CANARY_PROXY_PORT}</string>
    <key>MEMIND_PORTAL_TUNNEL_REMOTE_PORT</key><string>${CANARY_TUNNEL_REMOTE_PORT}</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-canary-tunnel.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-canary-tunnel.log</string>
</dict>
</plist>
EOF
  launchctl bootout "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
  launchctl bootstrap "${LAUNCHD_GUI}" "${CANARY_TUNNEL_PLIST}" >/dev/null
  launchctl enable "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
  launchctl kickstart -k "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null
}

restore_edge_to_stable() {
  edge_ssh \
    "set -euo pipefail
     mobile_backup_exists=0
     wechat_backup_exists=0
     [[ -f '${EDGE_MOBILE_BACKUP}' ]] && mobile_backup_exists=1
     [[ -f '${EDGE_WECHAT_BACKUP}' ]] && wechat_backup_exists=1
     [[ \"\${mobile_backup_exists}\" == \"\${wechat_backup_exists}\" ]]
     if [[ \"\${mobile_backup_exists}\" -eq 1 ]]; then
       test -f '${EDGE_MOBILE_BACKUP}.sha256'
       sha256sum -c '${EDGE_MOBILE_BACKUP}.sha256' >/dev/null
       cp '${EDGE_MOBILE_BACKUP}' '${EDGE_MOBILE_CONFIG}'
       cp '${EDGE_WECHAT_BACKUP}' '${EDGE_WECHAT_CONFIG}'
     fi
     grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}'
     grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}'
     ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}'
     ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}'
     nginx -t >/dev/null
     systemctl reload nginx
     curl -kfsS --max-time 15 \
       --resolve m.tkmind.cn:443:127.0.0.1 \
       https://m.tkmind.cn/api/status >/dev/null
     curl -kfsS --max-time 15 \
       --resolve wechat.m.tkmind.cn:443:127.0.0.1 \
       https://wechat.m.tkmind.cn/api/status >/dev/null"
}

stop_candidate_services() {
  launchctl bootout "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
  launchctl bootout "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null 2>&1 || true
  launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null 2>&1 || true
  launchctl bootout "${LAUNCHD_GUI}/${DEEPSEEK_COMPAT_LABEL}" >/dev/null 2>&1 || true
  "${DOCKER_BIN}" rm -f goosed-prod-canary >/dev/null 2>&1 || true
}

rollback() {
  local status=$?
  trap - ERR
  say "Canary rollout failed; restoring the direct stable edge upstream"
  rm -f "${STABLE_DIR}/.release-drain"
  if restore_edge_to_stable >/dev/null 2>&1; then
    stop_candidate_services
  else
    say "CRITICAL: edge restore could not be verified; keeping canary services running"
    say "Run scripts/rollback-portal-canary-prod.sh after checking the 105 nginx configs"
  fi
  exit "${status}"
}
trap rollback ERR

[[ -f "${BUNDLE}" && -f "${MANIFEST}" && -f "${SHA_FILE}" ]]
cd "${INCOMING_DIR}"
shasum -a 256 -c "$(basename "${SHA_FILE}")"

say "Create and verify the full stable backup"
COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
  -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")"
shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256"
shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256"
gzip -t "${FULL_BACKUP_TAR}"
tar -tzf "${FULL_BACKUP_TAR}" | grep '^Memind/.env$' >/dev/null

say "Create and verify the persisted-data backup"
persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")"
for item in .env MindSpace data users .tailscale public/plaza-covers logs; do
  if [[ -e "${STABLE_DIR}/${item}" ]]; then
    mkdir -p "${persist_tmp}/$(dirname "${item}")"
    if [[ "${item}" == ".tailscale" ]]; then
      mkdir -p "${persist_tmp}/${item}"
      (
        cd "${STABLE_DIR}/${item}"
        COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
      ) | (
        cd "${persist_tmp}/${item}"
        COPYFILE_DISABLE=1 tar -xf -
      )
    else
      cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/"
    fi
  fi
done
COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" .
rm -rf "${persist_tmp}"
shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256"
shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256"
gzip -t "${PERSIST_BACKUP_TAR}"
tar -tzf "${PERSIST_BACKUP_TAR}" | grep -E '^(\./)?\.env$' >/dev/null

say "Create and verify the active 105 nginx routing backup"
edge_ssh \
  "set -euo pipefail
   test -f '${EDGE_MOBILE_CONFIG}'
   test -f '${EDGE_WECHAT_CONFIG}'
   grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}'
   grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}'
   ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}'
   ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}'
   cp '${EDGE_MOBILE_CONFIG}' '${EDGE_MOBILE_BACKUP}'
   cp '${EDGE_WECHAT_CONFIG}' '${EDGE_WECHAT_BACKUP}'
   sha256sum '${EDGE_MOBILE_BACKUP}' '${EDGE_WECHAT_BACKUP}' \
     > '${EDGE_MOBILE_BACKUP}.sha256'
   sha256sum -c '${EDGE_MOBILE_BACKUP}.sha256'
   nginx -t >/dev/null"

say "Extract the source-free candidate runtime"
rm -rf "${CANDIDATE_DIR}"
mkdir -p "${CANDIDATE_DIR}"
tar -xzf "${BUNDLE}" -C "${CANDIDATE_DIR}"
cp "${MANIFEST}" "${CANDIDATE_DIR}/.release-manifest.txt"
chmod 755 \
  "${CANDIDATE_DIR}/scripts/run-memind-portal-candidate.sh" \
  "${CANDIDATE_DIR}/scripts/run-memind-canary-proxy-prod.sh" \
  "${CANDIDATE_DIR}/scripts/run-deepseek-compat-proxy-candidate.sh"

say "Stop an older canary without touching stable Portal 8081"
restore_edge_to_stable
stop_candidate_services

say "Start an isolated goosed candidate on 18015"
(
  cd "${GOOSED_DIR}"
  set -a
  # shellcheck disable=SC1091
  source .env
  set +a
  export CANDIDATE_RUNTIME_DIR="${CANDIDATE_DIR}"
  "${DOCKER_BIN}" compose -p goosed-prod \
    -f "${GOOSED_COMPOSE}" \
    -f "${CANDIDATE_DIR}/scripts/goosed-canary.compose.yml" \
    up -d --no-deps goosed-canary
)
for _ in $(seq 1 60); do
  if [[ "$(curl -skS -m 5 https://127.0.0.1:18015/status 2>/dev/null || true)" == "ok" ]]; then
    break
  fi
  sleep 2
done
[[ "$(curl -skS -m 5 https://127.0.0.1:18015/status 2>/dev/null || true)" == "ok" ]]
"${DOCKER_BIN}" exec goosed-prod-canary \
  sh -lc 'test -x /usr/local/bin/node && test -f /opt/portal/mindspace-sandbox-mcp.mjs'

say "Start the DeepSeek tool-round compatibility proxy on 18036"
cat > "${DEEPSEEK_COMPAT_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${DEEPSEEK_COMPAT_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${CANDIDATE_DIR}/scripts/run-deepseek-compat-proxy-candidate.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
    <key>MEMIND_CANARY_DEEPSEEK_PROXY_PORT</key><string>${DEEPSEEK_COMPAT_PORT}</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>ThrottleInterval</key><integer>5</integer>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-deepseek-compat-candidate.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-deepseek-compat-candidate.log</string>
</dict>
</plist>
EOF
launchctl bootstrap "${LAUNCHD_GUI}" "${DEEPSEEK_COMPAT_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${DEEPSEEK_COMPAT_LABEL}" >/dev/null
for _ in $(seq 1 30); do
  deepseek_compat_healthy && break
  sleep 1
done
deepseek_compat_healthy
"${DOCKER_BIN}" exec goosed-prod-canary \
  sh -lc "curl -fsS --max-time 5 http://host.docker.internal:${DEEPSEEK_COMPAT_PORT}/health \
    | grep -q '\"deepseekThinking\":\"disabled\"'"

say "Start the passive candidate Portal on 18081"
cat > "${CANDIDATE_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${PORTAL_CANDIDATE_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${CANDIDATE_DIR}/scripts/run-memind-portal-candidate.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
    <key>MEMIND_CANARY_RELEASE_ID</key><string>${RELEASE_ID}</string>
    <key>MEMIND_CANARY_CANDIDATE_PORT</key><string>18081</string>
    <key>MEMIND_CANARY_GOOSED_URL</key><string>https://127.0.0.1:18015</string>
    <key>MEMIND_CANARY_DEEPSEEK_PROXY_PORT</key><string>${DEEPSEEK_COMPAT_PORT}</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>ThrottleInterval</key><integer>5</integer>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-portal-candidate.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-portal-candidate.log</string>
</dict>
</plist>
EOF
launchctl bootstrap "${LAUNCHD_GUI}" "${CANDIDATE_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 60); do
  candidate_healthy && break
  sleep 2
done
candidate_healthy

say "Validate immutable canary selectors against the production identity database"
mkdir -p "$(dirname "${SECRET_FILE}")"
if [[ ! -s "${SECRET_FILE}" ]]; then
  umask 077
  /usr/bin/openssl rand -hex 32 > "${SECRET_FILE}"
fi
chmod 600 "${SECRET_FILE}"
(
  set -a
  # shellcheck disable=SC1091
  source "${STABLE_DIR}/.env"
  set +a
  export MEMIND_PORTAL_H5_ROOT="${STABLE_DIR}"
  export MEMIND_RELEASE_CANARY_USERNAMES="${CANARY_USERNAMES}"
  export MEMIND_RELEASE_CANARY_WECHAT_USER_IDS="${CANARY_WECHAT_USER_IDS}"
  /opt/homebrew/opt/node@24/bin/node "${CANDIDATE_DIR}/memind-canary-proxy.mjs" --check
)

say "Start the fail-closed identity router on 18082"
cat > "${PROXY_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${CANARY_PROXY_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${CANDIDATE_DIR}/scripts/run-memind-canary-proxy-prod.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
    <key>MEMIND_CANARY_SECRET_FILE</key><string>${SECRET_FILE}</string>
    <key>MEMIND_CANARY_PROXY_PORT</key><string>${CANARY_PROXY_PORT}</string>
    <key>MEMIND_RELEASE_CANARY_USERNAMES</key><string>${CANARY_USERNAMES}</string>
    <key>MEMIND_RELEASE_CANARY_WECHAT_USER_IDS</key><string>${CANARY_WECHAT_USER_IDS}</string>
    <key>MEMIND_CANARY_CANDIDATE_HEALTH_URLS</key><string>http://127.0.0.1:${DEEPSEEK_COMPAT_PORT}/health</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>ThrottleInterval</key><integer>5</integer>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-canary-proxy.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-canary-proxy.log</string>
</dict>
</plist>
EOF
launchctl bootstrap "${LAUNCHD_GUI}" "${PROXY_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null
sleep 2
diagnostic_secret="$(tr -d '\r\n' < "${SECRET_FILE}")"
diagnostic="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    "http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/health"
)"
printf '%s' "${diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (!status.ok || !status.stable || !status.candidate) process.exit(1);
    if (status.policy.usernames < 1 || status.policy.wechatUserIds < 1) process.exit(1);
  });
'
for probe_query in 'username=john' 'wechat_user_id=wx_ul610et8'; do
  route_probe="$(
    curl -fsS \
      -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
      "http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/route-probe?${probe_query}"
  )"
  printf '%s' "${route_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
    let input = "";
    process.stdin.on("data", (chunk) => { input += chunk; });
    process.stdin.on("end", () => {
      const status = JSON.parse(input);
      if (status.configuredTarget !== "candidate") process.exit(1);
      if (status.candidate !== true || status.route !== "candidate") process.exit(1);
    });
  '
done

say "Start the isolated 105 reverse tunnel on 19082"
write_canary_tunnel_plist
for _ in $(seq 1 30); do
  edge_code="$(
    edge_ssh \
      "curl -sS -o /dev/null -w '%{http_code}' \
       --max-time 5 http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status" \
      2>/dev/null || true
  )"
  [[ "${edge_code}" == "200" ]] && break
  sleep 2
done
[[ "${edge_code:-}" == "200" ]]

say "Drain active Agent runs for the bounded edge switch"
touch "${STABLE_DIR}/.release-drain"
deadline=$(( $(date +%s) + 120 ))
while (( $(date +%s) < deadline )); do
  queue="$(
    /opt/homebrew/opt/node@24/bin/node \
      "${STABLE_DIR}/scripts/agent-run-worker.mjs" --status 2>/dev/null || true
  )"
  counts="$(
    printf '%s' "${queue}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
      let input = "";
      process.stdin.on("data", (chunk) => { input += chunk; });
      process.stdin.on("end", () => {
        try {
          const queue = JSON.parse(input).queue || {};
          console.log(`${Number(queue.inFlight || 0)} ${Number(queue.pendingDispatches || 0)}`);
        } catch {
          console.log("unknown unknown");
        }
      });
    '
  )"
  if [[ "${counts}" == "0 0" ]]; then
    break
  fi
  sleep 2
done
[[ "${counts:-unknown unknown}" == "0 0" ]]

say "Switch the committed 105 nginx upstreams to the isolated canary tunnel"
# Unlike ordinary edge_ssh calls, this command intentionally receives only the
# bounded EDGE_SWITCH heredoc on stdin.
ssh -o BatchMode=yes -o ConnectTimeout=10 "${EDGE_HOST}" /bin/bash <<EDGE_SWITCH
set -euo pipefail
for config in '${EDGE_MOBILE_CONFIG}' '${EDGE_WECHAT_CONFIG}'; do
  before_count="\$(grep -cF 'proxy_pass http://58.38.22.103:8081;' "\${config}")"
  [[ "\${before_count}" -gt 0 ]]
  temp_config="\$(mktemp "\${config}.canary.XXXXXX")"
  sed 's#proxy_pass http://58\.38\.22\.103:8081;#proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};#g' \
    "\${config}" > "\${temp_config}"
  after_count="\$(grep -cF 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' "\${temp_config}")"
  [[ "\${after_count}" == "\${before_count}" ]]
  chown root:root "\${temp_config}"
  chmod 0644 "\${temp_config}"
  mv "\${temp_config}" "\${config}"
done
nginx -t
systemctl reload nginx
curl -fsS --max-time 10 http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status >/dev/null
curl -kfsS --max-time 15 \
  --resolve m.tkmind.cn:443:127.0.0.1 \
  https://m.tkmind.cn/api/status >/dev/null
curl -kfsS --max-time 15 \
  --resolve wechat.m.tkmind.cn:443:127.0.0.1 \
  https://wechat.m.tkmind.cn/api/status >/dev/null
EDGE_SWITCH
rm -f "${STABLE_DIR}/.release-drain"

say "Verify candidate failure automatically falls back to stable"
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 20); do
  if ! curl -fsS http://127.0.0.1:18081/api/status >/dev/null 2>&1; then
    break
  fi
  sleep 1
done
fallback_probe="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    "http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/route-probe?username=john"
)"
printf '%s' "${fallback_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (status.configuredTarget !== "candidate") process.exit(1);
    if (status.candidate !== false || status.route !== "stable") process.exit(1);
  });
'
edge_ssh \
  "curl -fsS --max-time 10 \
   http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status >/dev/null"
launchctl bootstrap "${LAUNCHD_GUI}" "${CANDIDATE_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 60); do
  candidate_healthy && break
  sleep 2
done
candidate_healthy
final_diagnostic="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    "http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/health"
)"
printf '%s' "${final_diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (!status.ok || !status.stable || !status.candidate) process.exit(1);
  });
'

trap - ERR
say "103 production canary is active"
printf 'release_id=%s\n' "${RELEASE_ID}"
printf 'candidate_dir=%s\n' "${CANDIDATE_DIR}"
printf 'full_backup=%s\n' "${FULL_BACKUP_TAR}"
printf 'persist_backup=%s\n' "${PERSIST_BACKUP_TAR}"
printf 'edge_mobile_backup=%s:%s\n' "${EDGE_HOST}" "${EDGE_MOBILE_BACKUP}"
printf 'edge_wechat_backup=%s:%s\n' "${EDGE_HOST}" "${EDGE_WECHAT_BACKUP}"
printf 'stable_health=http://127.0.0.1:8081/api/status\n'
printf 'proxy_health=http://127.0.0.1:%s/__memind_canary/health\n' "${CANARY_PROXY_PORT}"
printf 'edge_tunnel=http://127.0.0.1:%s/api/status\n' "${CANARY_TUNNEL_REMOTE_PORT}"
printf 'candidate_health=http://127.0.0.1:18081/api/status\n'
printf 'candidate_goosed=https://127.0.0.1:18015/status\n'
printf 'deepseek_compat_health=http://127.0.0.1:%s/health\n' "${DEEPSEEK_COMPAT_PORT}"
REMOTE_SCRIPT

say "103 canary release completed"
printf 'release_id=%s\n' "${RELEASE_ID}"
printf 'git_head=%s\n' "${FULL_SHA}"