memind/scripts/release-portal-canary-prod.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HOST="${STUDIO_HOST:-58.38.22.103}"
REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}"
STABLE_DIR="${REMOTE_ROOT}/Memind"
CANDIDATE_BASE="${REMOTE_ROOT}/Memind-candidates"
INCOMING_DIR="${REMOTE_ROOT}/incoming/memind-portal-canary"
BACKUP_DIR="${REMOTE_ROOT}/backups/memind"
GOOSED_DIR="${REMOTE_ROOT}/goosed-prod"
RUNTIME_ROOT="${ROOT}/.runtime/portal"
PORTAL_CANDIDATE_LABEL="cn.tkmind.memind-portal-candidate"
CANARY_PROXY_LABEL="cn.tkmind.memind-canary-proxy"
PORTAL_TUNNEL_LABEL="cn.tkmind.memind-portal-tunnel"
MEMIND_PORTAL_TUNNEL_HOST="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}"
MEMIND_PORTAL_TUNNEL_REMOTE_PORT="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}"
CANARY_USERNAMES="john"
CANARY_WECHAT_USER_IDS="wx_ul610et8"
RELEASE_TS="$(date +%Y%m%d-%H%M%S)"
FULL_SHA="$(git -C "${ROOT}" rev-parse HEAD)"
SHORT_SHA="${FULL_SHA:0:7}"
RELEASE_ID="${RELEASE_TS}-${SHORT_SHA}"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/memind-portal-canary-release.XXXXXX")"
BUNDLE_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.tar.gz"
MANIFEST_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.manifest.txt"
SHA_PATH="${TMP_DIR}/memind-portal-canary-${RELEASE_ID}.sha256"
DRY_RUN=0
AUTO_YES=0

cleanup() {
  rm -rf "${TMP_DIR}"
}
trap cleanup EXIT

usage() {
  cat <<'EOF'
Usage:
  bash scripts/release-portal-canary-prod.sh [--dry-run] [--yes]

Deploys a candidate Portal beside the stable 103 runtime:
  stable Portal     127.0.0.1:8081
  canary router     127.0.0.1:18080
  candidate Portal  127.0.0.1:18081
  candidate goosed  127.0.0.1:18015

The stable runtime is not replaced. The 105 reverse tunnel moves to the canary
router only after backups, candidate health, immutable identity checks, and
automatic fallback checks pass.
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --dry-run) DRY_RUN=1 ;;
    --yes|-y) AUTO_YES=1 ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 1
      ;;
  esac
  shift
done

for bypass in \
  ALLOW_PORTAL_RELEASE_SCOPE_BYPASS \
  ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES \
  MEMIND_RELEASE_SKIP_GATE; do
  if [[ "${!bypass:-0}" == "1" ]]; then
    echo "Production canary release forbids ${bypass}." >&2
    exit 1
  fi
done

say() {
  printf '\n[%s] %s\n' "$(date +%H:%M:%S)" "$*"
}

need_cmd() {
  command -v "$1" >/dev/null 2>&1 || {
    echo "Missing command: $1" >&2
    exit 1
  }
}

need_cmd ssh
need_cmd scp
need_cmd tar
need_cmd shasum

branch="$(git -C "${ROOT}" branch --show-current)"
if [[ "${branch}" != "main" ]]; then
  echo "Canary production release requires main, got: ${branch:-detached}" >&2
  exit 1
fi
git -C "${ROOT}" fetch origin --prune
if [[ "$(git -C "${ROOT}" rev-parse HEAD)" != "$(git -C "${ROOT}" rev-parse origin/main)" ]]; then
  echo "Canary production release requires HEAD to equal origin/main." >&2
  exit 1
fi
ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh" --skip-fetch

say "Run release source guards"
(
  cd "${ROOT}"
  npm test -- --test-name-pattern='publish|space|billing' >/dev/null
  npm run verify:mindspace-publish-guards >/dev/null
  npm run verify:mindspace-page-sync-guards >/dev/null
  npm run verify:h5-session-patches >/dev/null
  npm run verify:page-data >/dev/null
  npm run check:mindspace-public-links >/dev/null
)

required_runtime_paths=(
  server.mjs
  memind-canary-proxy.mjs
  wechat-mp.bundle.mjs
  mindspace-sandbox-mcp.mjs
  tkmind-search-mcp.mjs
  tkmind-excel-mcp.mjs
  dist
  package.json
  scripts/run-memind-portal-candidate.sh
  scripts/run-memind-canary-proxy-prod.sh
  scripts/goosed-canary.compose.yml
  scripts/memind-portal-tunnel.sh
)
for relative in "${required_runtime_paths[@]}"; do
  [[ -e "${RUNTIME_ROOT}/${relative}" ]] || {
    echo "Candidate runtime is missing ${relative}; rebuild and rerun the complete Gate." >&2
    exit 1
  }
done

say "Verify the complete Gate report against the exact candidate artifact"
node "${ROOT}/scripts/verify-release-gate-report.mjs" --artifact "${RUNTIME_ROOT}"

say "Package the verified candidate artifact"
tar -czf "${BUNDLE_PATH}" -C "${RUNTIME_ROOT}" .
bundle_sha="$(shasum -a 256 "${BUNDLE_PATH}" | awk '{print $1}')"
printf '%s  %s\n' "${bundle_sha}" "$(basename "${BUNDLE_PATH}")" > "${SHA_PATH}"
{
  echo "release_id=${RELEASE_ID}"
  echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')"
  echo "git_head=${FULL_SHA}"
  echo "git_branch=${branch}"
  echo "artifact_tree=.runtime/portal"
  echo "artifact_bundle_sha256=${bundle_sha}"
  echo "routing=stable:8081,proxy:18080,candidate:18081,goosed-canary:18015"
  echo "canary_usernames=${CANARY_USERNAMES}"
  echo "canary_wechat_user_ids=${CANARY_WECHAT_USER_IDS}"
  echo "manual_environment_changes=none"
} > "${MANIFEST_PATH}"

if [[ "${DRY_RUN}" -eq 1 ]]; then
  say "Dry-run complete; no 103 connection or production write occurred"
  ls -lh "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}"
  exit 0
fi

say "Run 103 read-only preflight"
ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" \
  "test -d '${STABLE_DIR}' \
   && test -f '${STABLE_DIR}/.env' \
   && test -f '${GOOSED_DIR}/docker-compose.prod.yml' \
   && curl -fsS http://127.0.0.1:8081/api/status >/dev/null \
   && /opt/homebrew/bin/docker inspect goosed-prod-1 >/dev/null \
   && test \"\$(df -Pk '${REMOTE_ROOT}' | awk 'NR==2 {print \$4}')\" -gt 10485760"

if [[ "${AUTO_YES}" -ne 1 ]]; then
  say "Production canary confirmation"
  echo "Target host: ${HOST}"
  echo "Release: ${RELEASE_ID}"
  echo "Commit: ${FULL_SHA}"
  echo "Stable remains on 8081; selected immutable identities route to candidate 18081."
  read -r -p "Continue with the 103 canary release? [y/N] " confirm </dev/tty
  [[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
fi

say "Upload the verified candidate bundle"
ssh -o BatchMode=yes "${HOST}" "mkdir -p '${INCOMING_DIR}' '${BACKUP_DIR}' '${CANDIDATE_BASE}'"
scp -q "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}" "${HOST}:${INCOMING_DIR}/"

say "Install the isolated candidate and identity router on 103"
ssh -o BatchMode=yes "${HOST}" \
  "RELEASE_ID='${RELEASE_ID}' \
   STABLE_DIR='${STABLE_DIR}' \
   CANDIDATE_BASE='${CANDIDATE_BASE}' \
   INCOMING_DIR='${INCOMING_DIR}' \
   BACKUP_DIR='${BACKUP_DIR}' \
   GOOSED_DIR='${GOOSED_DIR}' \
   PORTAL_CANDIDATE_LABEL='${PORTAL_CANDIDATE_LABEL}' \
   CANARY_PROXY_LABEL='${CANARY_PROXY_LABEL}' \
   PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' \
   MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' \
   MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' \
   CANARY_USERNAMES='${CANARY_USERNAMES}' \
   CANARY_WECHAT_USER_IDS='${CANARY_WECHAT_USER_IDS}' \
   /bin/bash" <<'REMOTE_SCRIPT'
set -euo pipefail

LAUNCHD_GUI="gui/$(id -u)"
CANDIDATE_DIR="${CANDIDATE_BASE}/${RELEASE_ID}"
BUNDLE="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.tar.gz"
MANIFEST="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.manifest.txt"
SHA_FILE="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.sha256"
FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before-canary.tar.gz"
PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before-canary.tar.gz"
TUNNEL_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_TUNNEL_LABEL}.plist"
TUNNEL_PLIST_BACKUP="${BACKUP_DIR}/${PORTAL_TUNNEL_LABEL}-${RELEASE_ID}.plist"
CANDIDATE_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_CANDIDATE_LABEL}.plist"
PROXY_PLIST="${HOME}/Library/LaunchAgents/${CANARY_PROXY_LABEL}.plist"
SECRET_FILE="${HOME}/.config/memind/canary-router.secret"
DOCKER_BIN="/opt/homebrew/bin/docker"
GOOSED_COMPOSE="${GOOSED_DIR}/docker-compose.prod.yml"

say() {
  printf '\n[remote %s] %s\n' "$(date +%H:%M:%S)" "$*"
}

candidate_healthy() {
  local headers code
  headers="$(mktemp)"
  code="$(
    curl -sS -D "${headers}" -o /dev/null -w '%{http_code}' \
      http://127.0.0.1:18081/api/status 2>/dev/null || true
  )"
  if [[ "${code}" == "200" ]] \
    && grep -qi '^x-memind-runtime-role: candidate' "${headers}"; then
    rm -f "${headers}"
    return 0
  fi
  rm -f "${headers}"
  return 1
}

write_tunnel_plist() {
  local local_port="$1"
  cat > "${TUNNEL_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${PORTAL_TUNNEL_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${STABLE_DIR}/scripts/memind-portal-tunnel.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_PORTAL_TUNNEL_HOST</key><string>${MEMIND_PORTAL_TUNNEL_HOST}</string>
    <key>MEMIND_PORTAL_TUNNEL_LOCAL_PORT</key><string>${local_port}</string>
    <key>MEMIND_PORTAL_TUNNEL_REMOTE_PORT</key><string>${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
</dict>
</plist>
EOF
  launchctl bootout "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null 2>&1 || true
  launchctl bootstrap "${LAUNCHD_GUI}" "${TUNNEL_PLIST}" >/dev/null
  launchctl enable "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null 2>&1 || true
  launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null
}

stop_candidate_services() {
  launchctl bootout "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null 2>&1 || true
  launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null 2>&1 || true
  "${DOCKER_BIN}" rm -f goosed-prod-canary >/dev/null 2>&1 || true
}

rollback() {
  local status=$?
  trap - ERR
  say "Canary rollout failed; restoring stable-only tunnel"
  rm -f "${STABLE_DIR}/.release-drain"
  write_tunnel_plist 8081 >/dev/null 2>&1 || true
  stop_candidate_services
  exit "${status}"
}
trap rollback ERR

[[ -f "${BUNDLE}" && -f "${MANIFEST}" && -f "${SHA_FILE}" ]]
cd "${INCOMING_DIR}"
shasum -a 256 -c "$(basename "${SHA_FILE}")"

say "Create and verify the full stable backup"
COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
  -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")"
shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256"
shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256"
gzip -t "${FULL_BACKUP_TAR}"
tar -tzf "${FULL_BACKUP_TAR}" | grep '^Memind/.env$' >/dev/null

say "Create and verify the persisted-data backup"
persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")"
for item in .env MindSpace data users .tailscale public/plaza-covers logs; do
  if [[ -e "${STABLE_DIR}/${item}" ]]; then
    mkdir -p "${persist_tmp}/$(dirname "${item}")"
    if [[ "${item}" == ".tailscale" ]]; then
      mkdir -p "${persist_tmp}/${item}"
      (
        cd "${STABLE_DIR}/${item}"
        COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
      ) | (
        cd "${persist_tmp}/${item}"
        COPYFILE_DISABLE=1 tar -xf -
      )
    else
      cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/"
    fi
  fi
done
COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" .
rm -rf "${persist_tmp}"
shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256"
shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256"
gzip -t "${PERSIST_BACKUP_TAR}"
tar -tzf "${PERSIST_BACKUP_TAR}" | grep -E '^(\./)?\.env$' >/dev/null

if [[ -f "${TUNNEL_PLIST}" ]]; then
  cp "${TUNNEL_PLIST}" "${TUNNEL_PLIST_BACKUP}"
fi

say "Extract the source-free candidate runtime"
rm -rf "${CANDIDATE_DIR}"
mkdir -p "${CANDIDATE_DIR}"
tar -xzf "${BUNDLE}" -C "${CANDIDATE_DIR}"
cp "${MANIFEST}" "${CANDIDATE_DIR}/.release-manifest.txt"
chmod 755 \
  "${CANDIDATE_DIR}/scripts/run-memind-portal-candidate.sh" \
  "${CANDIDATE_DIR}/scripts/run-memind-canary-proxy-prod.sh"

say "Stop an older canary without touching stable Portal 8081"
if curl -fsS http://127.0.0.1:18080/api/status >/dev/null 2>&1; then
  write_tunnel_plist 8081
  sleep 2
fi
stop_candidate_services

say "Start an isolated goosed candidate on 18015"
(
  cd "${GOOSED_DIR}"
  set -a
  # shellcheck disable=SC1091
  source .env
  set +a
  export CANDIDATE_RUNTIME_DIR="${CANDIDATE_DIR}"
  "${DOCKER_BIN}" compose -p goosed-prod \
    -f "${GOOSED_COMPOSE}" \
    -f "${CANDIDATE_DIR}/scripts/goosed-canary.compose.yml" \
    up -d --no-deps goosed-canary
)
for _ in $(seq 1 60); do
  if [[ "$(curl -skS -m 5 https://127.0.0.1:18015/status 2>/dev/null || true)" == "ok" ]]; then
    break
  fi
  sleep 2
done
[[ "$(curl -skS -m 5 https://127.0.0.1:18015/status 2>/dev/null || true)" == "ok" ]]
"${DOCKER_BIN}" exec goosed-prod-canary \
  sh -lc 'test -x /usr/local/bin/node && test -f /opt/portal/mindspace-sandbox-mcp.mjs'

say "Start the passive candidate Portal on 18081"
cat > "${CANDIDATE_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${PORTAL_CANDIDATE_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${CANDIDATE_DIR}/scripts/run-memind-portal-candidate.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
    <key>MEMIND_CANARY_RELEASE_ID</key><string>${RELEASE_ID}</string>
    <key>MEMIND_CANARY_CANDIDATE_PORT</key><string>18081</string>
    <key>MEMIND_CANARY_GOOSED_URL</key><string>https://127.0.0.1:18015</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>ThrottleInterval</key><integer>5</integer>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-portal-candidate.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-portal-candidate.log</string>
</dict>
</plist>
EOF
launchctl bootstrap "${LAUNCHD_GUI}" "${CANDIDATE_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 60); do
  candidate_healthy && break
  sleep 2
done
candidate_healthy

say "Validate immutable canary selectors against the production identity database"
mkdir -p "$(dirname "${SECRET_FILE}")"
if [[ ! -s "${SECRET_FILE}" ]]; then
  umask 077
  /usr/bin/openssl rand -hex 32 > "${SECRET_FILE}"
fi
chmod 600 "${SECRET_FILE}"
(
  set -a
  # shellcheck disable=SC1091
  source "${STABLE_DIR}/.env"
  set +a
  export MEMIND_PORTAL_H5_ROOT="${STABLE_DIR}"
  export MEMIND_RELEASE_CANARY_USERNAMES="${CANARY_USERNAMES}"
  export MEMIND_RELEASE_CANARY_WECHAT_USER_IDS="${CANARY_WECHAT_USER_IDS}"
  /opt/homebrew/opt/node@24/bin/node "${CANDIDATE_DIR}/memind-canary-proxy.mjs" --check
)

say "Start the fail-closed identity router on 18080"
cat > "${PROXY_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
  <key>Label</key><string>${CANARY_PROXY_LABEL}</string>
  <key>ProgramArguments</key>
  <array><string>${CANDIDATE_DIR}/scripts/run-memind-canary-proxy-prod.sh</string></array>
  <key>EnvironmentVariables</key>
  <dict>
    <key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
    <key>MEMIND_CANARY_SECRET_FILE</key><string>${SECRET_FILE}</string>
    <key>MEMIND_RELEASE_CANARY_USERNAMES</key><string>${CANARY_USERNAMES}</string>
    <key>MEMIND_RELEASE_CANARY_WECHAT_USER_IDS</key><string>${CANARY_WECHAT_USER_IDS}</string>
  </dict>
  <key>RunAtLoad</key><true/>
  <key>KeepAlive</key><true/>
  <key>ThrottleInterval</key><integer>5</integer>
  <key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-canary-proxy.log</string>
  <key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-canary-proxy.log</string>
</dict>
</plist>
EOF
launchctl bootstrap "${LAUNCHD_GUI}" "${PROXY_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null
sleep 2
diagnostic_secret="$(tr -d '\r\n' < "${SECRET_FILE}")"
diagnostic="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    http://127.0.0.1:18080/__memind_canary/health
)"
printf '%s' "${diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (!status.ok || !status.stable || !status.candidate) process.exit(1);
    if (status.policy.usernames < 1 || status.policy.wechatUserIds < 1) process.exit(1);
  });
'
for probe_query in 'username=john' 'wechat_user_id=wx_ul610et8'; do
  route_probe="$(
    curl -fsS \
      -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
      "http://127.0.0.1:18080/__memind_canary/route-probe?${probe_query}"
  )"
  printf '%s' "${route_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
    let input = "";
    process.stdin.on("data", (chunk) => { input += chunk; });
    process.stdin.on("end", () => {
      const status = JSON.parse(input);
      if (status.configuredTarget !== "candidate") process.exit(1);
      if (status.candidate !== true || status.route !== "candidate") process.exit(1);
    });
  '
done

say "Drain active Agent runs for the bounded tunnel switch"
touch "${STABLE_DIR}/.release-drain"
deadline=$(( $(date +%s) + 120 ))
while (( $(date +%s) < deadline )); do
  queue="$(
    /opt/homebrew/opt/node@24/bin/node \
      "${STABLE_DIR}/scripts/agent-run-worker.mjs" --status 2>/dev/null || true
  )"
  counts="$(
    printf '%s' "${queue}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
      let input = "";
      process.stdin.on("data", (chunk) => { input += chunk; });
      process.stdin.on("end", () => {
        try {
          const queue = JSON.parse(input).queue || {};
          console.log(`${Number(queue.inFlight || 0)} ${Number(queue.pendingDispatches || 0)}`);
        } catch {
          console.log("unknown unknown");
        }
      });
    '
  )"
  if [[ "${counts}" == "0 0" ]]; then
    break
  fi
  sleep 2
done
[[ "${counts:-unknown unknown}" == "0 0" ]]

say "Switch only the reverse tunnel from stable 8081 to identity router 18080"
write_tunnel_plist 18080
sleep 2
remote_code="$(
  ssh -o BatchMode=yes -o ConnectTimeout=10 "${MEMIND_PORTAL_TUNNEL_HOST}" \
    "curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}/api/status" \
    2>/dev/null || true
)"
[[ "${remote_code}" == "200" ]]
rm -f "${STABLE_DIR}/.release-drain"

say "Verify candidate failure automatically falls back to stable"
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 20); do
  if ! curl -fsS http://127.0.0.1:18081/api/status >/dev/null 2>&1; then
    break
  fi
  sleep 1
done
fallback_probe="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    'http://127.0.0.1:18080/__memind_canary/route-probe?username=john'
)"
printf '%s' "${fallback_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (status.configuredTarget !== "candidate") process.exit(1);
    if (status.candidate !== false || status.route !== "stable") process.exit(1);
  });
'
launchctl bootstrap "${LAUNCHD_GUI}" "${CANDIDATE_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 60); do
  candidate_healthy && break
  sleep 2
done
candidate_healthy
final_diagnostic="$(
  curl -fsS \
    -H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
    http://127.0.0.1:18080/__memind_canary/health
)"
printf '%s' "${final_diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
  let input = "";
  process.stdin.on("data", (chunk) => { input += chunk; });
  process.stdin.on("end", () => {
    const status = JSON.parse(input);
    if (!status.ok || !status.stable || !status.candidate) process.exit(1);
  });
'

trap - ERR
say "103 production canary is active"
printf 'release_id=%s\n' "${RELEASE_ID}"
printf 'candidate_dir=%s\n' "${CANDIDATE_DIR}"
printf 'full_backup=%s\n' "${FULL_BACKUP_TAR}"
printf 'persist_backup=%s\n' "${PERSIST_BACKUP_TAR}"
printf 'stable_health=http://127.0.0.1:8081/api/status\n'
printf 'proxy_health=http://127.0.0.1:18080/__memind_canary/health\n'
printf 'candidate_health=http://127.0.0.1:18081/api/status\n'
printf 'candidate_goosed=https://127.0.0.1:18015/status\n'
REMOTE_SCRIPT

say "103 canary release completed"
printf 'release_id=%s\n' "${RELEASE_ID}"
printf 'git_head=%s\n' "${FULL_SHA}"