tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: route portal canary through isolated edge tunnel
Memind CI / Test, build, and release guards (push) Has been cancelled
Details

Browse Source
This commit is contained in:
john
2026-07-26 20:30:56 +08:00
parent 286069449b
commit 473ad80882
10 changed files with 306 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/release-portal-canary-prod.sh
+157 -54
View File
@@ -12,9 +12,12 @@ GOOSED_DIR="${REMOTE_ROOT}/goosed-prod"
RUNTIME_ROOT="${ROOT}/.runtime/portal"
PORTAL_CANDIDATE_LABEL="cn.tkmind.memind-portal-candidate"
CANARY_PROXY_LABEL="cn.tkmind.memind-canary-proxy"
PORTAL_TUNNEL_LABEL="cn.tkmind.memind-portal-tunnel"
MEMIND_PORTAL_TUNNEL_HOST="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}"
MEMIND_PORTAL_TUNNEL_REMOTE_PORT="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}"
CANARY_TUNNEL_LABEL="cn.tkmind.memind-canary-tunnel"
EDGE_HOST="${MEMIND_CANARY_EDGE_HOST:-ssh105-public}"
CANARY_PROXY_PORT=18082
CANARY_TUNNEL_REMOTE_PORT=19082
EDGE_MOBILE_CONFIG="/etc/nginx/conf.d/m.tkmind.cn.conf"
EDGE_WECHAT_CONFIG="/etc/nginx/conf.d/wechat.m.tkmind.cn.conf"
CANARY_USERNAMES="john"
CANARY_WECHAT_USER_IDS="wx_ul610et8"
RELEASE_TS="$(date +%Y%m%d-%H%M%S)"
@@ -40,13 +43,14 @@ Usage:
Deploys a candidate Portal beside the stable 103 runtime:
stable Portal 127.0.0.1:8081
canary router 127.0.0.1:18080
canary router 127.0.0.1:18082
candidate Portal 127.0.0.1:18081
candidate goosed 127.0.0.1:18015
The stable runtime is not replaced. The 105 reverse tunnel moves to the canary
router only after backups, candidate health, immutable identity checks, and
automatic fallback checks pass.
The stable runtime is not replaced. A dedicated reverse tunnel exposes the
router only to 105 at 127.0.0.1:19082. The committed release workflow updates
the two active 105 nginx upstreams only after backups, candidate health, and
immutable identity checks pass, then proves automatic fallback before success.
EOF
}
@@ -151,7 +155,7 @@ printf '%s %s\n' "${bundle_sha}" "$(basename "${BUNDLE_PATH}")" > "${SHA_PATH}"
echo "git_branch=${branch}"
echo "artifact_tree=.runtime/portal"
echo "artifact_bundle_sha256=${bundle_sha}"
echo "routing=stable:8081,proxy:18080,candidate:18081,goosed-canary:18015"
echo "routing=stable:8081,proxy:18082,edge-tunnel:19082,candidate:18081,goosed-canary:18015"
echo "canary_usernames=${CANARY_USERNAMES}"
echo "canary_wechat_user_ids=${CANARY_WECHAT_USER_IDS}"
echo "manual_environment_changes=none"
@@ -169,15 +173,28 @@ ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" \
&& test -f '${STABLE_DIR}/.env' \
&& test -f '${GOOSED_DIR}/docker-compose.prod.yml' \
&& curl -fsS http://127.0.0.1:8081/api/status >/dev/null \
&& ! lsof -nP -iTCP:${CANARY_PROXY_PORT} -sTCP:LISTEN >/dev/null 2>&1 \
&& ! lsof -nP -iTCP:18081 -sTCP:LISTEN >/dev/null 2>&1 \
&& ! lsof -nP -iTCP:18015 -sTCP:LISTEN >/dev/null 2>&1 \
&& /opt/homebrew/bin/docker inspect goosed-prod-1 >/dev/null \
&& test \"\$(df -Pk '${REMOTE_ROOT}' | awk 'NR==2 {print \$4}')\" -gt 10485760"
&& test \"\$(df -Pk '${REMOTE_ROOT}' | awk 'NR==2 {print \$4}')\" -gt 10485760 \
&& ssh -o BatchMode=yes -o ConnectTimeout=10 '${EDGE_HOST}' \
\"test -f '${EDGE_MOBILE_CONFIG}' \
&& test -f '${EDGE_WECHAT_CONFIG}' \
&& grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}' \
&& grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}' \
&& ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}' \
&& ! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}' \
&& nginx -t >/dev/null 2>&1 \
&& ! lsof -nP -iTCP:${CANARY_TUNNEL_REMOTE_PORT} -sTCP:LISTEN >/dev/null 2>&1\""
if [[ "${AUTO_YES}" -ne 1 ]]; then
say "Production canary confirmation"
echo "Target host: ${HOST}"
echo "Release: ${RELEASE_ID}"
echo "Commit: ${FULL_SHA}"
echo "Stable remains on 8081; selected immutable identities route to candidate 18081."
echo "Stable remains on 8081; 105 will use the isolated 19082 canary tunnel."
echo "Selected immutable identities route to candidate 18081."
read -r -p "Continue with the 103 canary release? [y/N] " confirm </dev/tty
[[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
fi
@@ -196,9 +213,12 @@ ssh -o BatchMode=yes "${HOST}" \
GOOSED_DIR='${GOOSED_DIR}' \
PORTAL_CANDIDATE_LABEL='${PORTAL_CANDIDATE_LABEL}' \
CANARY_PROXY_LABEL='${CANARY_PROXY_LABEL}' \
PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' \
MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' \
MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' \
CANARY_TUNNEL_LABEL='${CANARY_TUNNEL_LABEL}' \
EDGE_HOST='${EDGE_HOST}' \
CANARY_PROXY_PORT='${CANARY_PROXY_PORT}' \
CANARY_TUNNEL_REMOTE_PORT='${CANARY_TUNNEL_REMOTE_PORT}' \
EDGE_MOBILE_CONFIG='${EDGE_MOBILE_CONFIG}' \
EDGE_WECHAT_CONFIG='${EDGE_WECHAT_CONFIG}' \
CANARY_USERNAMES='${CANARY_USERNAMES}' \
CANARY_WECHAT_USER_IDS='${CANARY_WECHAT_USER_IDS}' \
/bin/bash" <<'REMOTE_SCRIPT'
@@ -211,13 +231,14 @@ MANIFEST="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.manifest.txt"
SHA_FILE="${INCOMING_DIR}/memind-portal-canary-${RELEASE_ID}.sha256"
FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before-canary.tar.gz"
PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before-canary.tar.gz"
TUNNEL_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_TUNNEL_LABEL}.plist"
TUNNEL_PLIST_BACKUP="${BACKUP_DIR}/${PORTAL_TUNNEL_LABEL}-${RELEASE_ID}.plist"
CANDIDATE_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_CANDIDATE_LABEL}.plist"
PROXY_PLIST="${HOME}/Library/LaunchAgents/${CANARY_PROXY_LABEL}.plist"
CANARY_TUNNEL_PLIST="${HOME}/Library/LaunchAgents/${CANARY_TUNNEL_LABEL}.plist"
SECRET_FILE="${HOME}/.config/memind/canary-router.secret"
DOCKER_BIN="/opt/homebrew/bin/docker"
GOOSED_COMPOSE="${GOOSED_DIR}/docker-compose.prod.yml"
EDGE_MOBILE_BACKUP="${EDGE_MOBILE_CONFIG}.before-canary-${RELEASE_ID}"
EDGE_WECHAT_BACKUP="${EDGE_WECHAT_CONFIG}.before-canary-${RELEASE_ID}"
say() {
printf '\n[remote %s] %s\n' "$(date +%H:%M:%S)" "$*"
@@ -239,36 +260,68 @@ candidate_healthy() {
return 1
}
write_tunnel_plist() {
local local_port="$1"
cat > "${TUNNEL_PLIST}" <<EOF
edge_ssh() {
ssh -o BatchMode=yes -o ConnectTimeout=10 "${EDGE_HOST}" "$@"
}
write_canary_tunnel_plist() {
cat > "${CANARY_TUNNEL_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key><string>${PORTAL_TUNNEL_LABEL}</string>
<key>Label</key><string>${CANARY_TUNNEL_LABEL}</string>
<key>ProgramArguments</key>
<array><string>${STABLE_DIR}/scripts/memind-portal-tunnel.sh</string></array>
<key>EnvironmentVariables</key>
<dict>
<key>MEMIND_PORTAL_TUNNEL_HOST</key><string>${MEMIND_PORTAL_TUNNEL_HOST}</string>
<key>MEMIND_PORTAL_TUNNEL_LOCAL_PORT</key><string>${local_port}</string>
<key>MEMIND_PORTAL_TUNNEL_REMOTE_PORT</key><string>${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}</string>
<key>MEMIND_PORTAL_TUNNEL_HOST</key><string>${EDGE_HOST}</string>
<key>MEMIND_PORTAL_TUNNEL_LOCAL_PORT</key><string>${CANARY_PROXY_PORT}</string>
<key>MEMIND_PORTAL_TUNNEL_REMOTE_PORT</key><string>${CANARY_TUNNEL_REMOTE_PORT}</string>
</dict>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-canary-tunnel.log</string>
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-canary-tunnel.log</string>
</dict>
</plist>
EOF
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl bootstrap "${LAUNCHD_GUI}" "${TUNNEL_PLIST}" >/dev/null
launchctl enable "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null
launchctl bootout "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl bootstrap "${LAUNCHD_GUI}" "${CANARY_TUNNEL_PLIST}" >/dev/null
launchctl enable "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl kickstart -k "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null
}
restore_edge_to_stable() {
edge_ssh \
"set -euo pipefail
mobile_backup_exists=0
wechat_backup_exists=0
[[ -f '${EDGE_MOBILE_BACKUP}' ]] && mobile_backup_exists=1
[[ -f '${EDGE_WECHAT_BACKUP}' ]] && wechat_backup_exists=1
[[ \"\${mobile_backup_exists}\" == \"\${wechat_backup_exists}\" ]]
if [[ \"\${mobile_backup_exists}\" -eq 1 ]]; then
test -f '${EDGE_MOBILE_BACKUP}.sha256'
sha256sum -c '${EDGE_MOBILE_BACKUP}.sha256' >/dev/null
cp '${EDGE_MOBILE_BACKUP}' '${EDGE_MOBILE_CONFIG}'
cp '${EDGE_WECHAT_BACKUP}' '${EDGE_WECHAT_CONFIG}'
fi
grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}'
grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}'
! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}'
! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}'
nginx -t >/dev/null
systemctl reload nginx
curl -kfsS --max-time 15 \
--resolve m.tkmind.cn:443:127.0.0.1 \
https://m.tkmind.cn/api/status >/dev/null
curl -kfsS --max-time 15 \
--resolve wechat.m.tkmind.cn:443:127.0.0.1 \
https://wechat.m.tkmind.cn/api/status >/dev/null"
}
stop_candidate_services() {
launchctl bootout "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl bootout "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null 2>&1 || true
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null 2>&1 || true
"${DOCKER_BIN}" rm -f goosed-prod-canary >/dev/null 2>&1 || true
@@ -277,10 +330,14 @@ stop_candidate_services() {
rollback() {
local status=$?
trap - ERR
say "Canary rollout failed; restoring stable-only tunnel"
say "Canary rollout failed; restoring the direct stable edge upstream"
rm -f "${STABLE_DIR}/.release-drain"
write_tunnel_plist 8081 >/dev/null 2>&1 || true
stop_candidate_services
if restore_edge_to_stable >/dev/null 2>&1; then
stop_candidate_services
else
say "CRITICAL: edge restore could not be verified; keeping canary services running"
say "Run scripts/rollback-portal-canary-prod.sh after checking the 105 nginx configs"
fi
exit "${status}"
}
trap rollback ERR
@@ -323,9 +380,21 @@ shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256"
gzip -t "${PERSIST_BACKUP_TAR}"
tar -tzf "${PERSIST_BACKUP_TAR}" | grep -E '^(\./)?\.env$' >/dev/null
if [[ -f "${TUNNEL_PLIST}" ]]; then
cp "${TUNNEL_PLIST}" "${TUNNEL_PLIST_BACKUP}"
fi
say "Create and verify the active 105 nginx routing backup"
edge_ssh \
"set -euo pipefail
test -f '${EDGE_MOBILE_CONFIG}'
test -f '${EDGE_WECHAT_CONFIG}'
grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_MOBILE_CONFIG}'
grep -q 'proxy_pass http://58.38.22.103:8081;' '${EDGE_WECHAT_CONFIG}'
! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_MOBILE_CONFIG}'
! grep -q 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' '${EDGE_WECHAT_CONFIG}'
cp '${EDGE_MOBILE_CONFIG}' '${EDGE_MOBILE_BACKUP}'
cp '${EDGE_WECHAT_CONFIG}' '${EDGE_WECHAT_BACKUP}'
sha256sum '${EDGE_MOBILE_BACKUP}' '${EDGE_WECHAT_BACKUP}' \
> '${EDGE_MOBILE_BACKUP}.sha256'
sha256sum -c '${EDGE_MOBILE_BACKUP}.sha256'
nginx -t >/dev/null"
say "Extract the source-free candidate runtime"
rm -rf "${CANDIDATE_DIR}"
@@ -337,10 +406,7 @@ chmod 755 \
"${CANDIDATE_DIR}/scripts/run-memind-canary-proxy-prod.sh"
say "Stop an older canary without touching stable Portal 8081"
if curl -fsS http://127.0.0.1:18080/api/status >/dev/null 2>&1; then
write_tunnel_plist 8081
sleep 2
fi
restore_edge_to_stable
stop_candidate_services
say "Start an isolated goosed candidate on 18015"
@@ -416,7 +482,7 @@ chmod 600 "${SECRET_FILE}"
/opt/homebrew/opt/node@24/bin/node "${CANDIDATE_DIR}/memind-canary-proxy.mjs" --check
)
say "Start the fail-closed identity router on 18080"
say "Start the fail-closed identity router on 18082"
cat > "${PROXY_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -429,6 +495,7 @@ cat > "${PROXY_PLIST}" <<EOF
<dict>
<key>MEMIND_CANARY_STABLE_ROOT</key><string>${STABLE_DIR}</string>
<key>MEMIND_CANARY_SECRET_FILE</key><string>${SECRET_FILE}</string>
<key>MEMIND_CANARY_PROXY_PORT</key><string>${CANARY_PROXY_PORT}</string>
<key>MEMIND_RELEASE_CANARY_USERNAMES</key><string>${CANARY_USERNAMES}</string>
<key>MEMIND_RELEASE_CANARY_WECHAT_USER_IDS</key><string>${CANARY_WECHAT_USER_IDS}</string>
</dict>
@@ -447,7 +514,7 @@ diagnostic_secret="$(tr -d '\r\n' < "${SECRET_FILE}")"
diagnostic="$(
curl -fsS \
-H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
http://127.0.0.1:18080/__memind_canary/health
"http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/health"
)"
printf '%s' "${diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
let input = "";
@@ -462,7 +529,7 @@ for probe_query in 'username=john' 'wechat_user_id=wx_ul610et8'; do
route_probe="$(
curl -fsS \
-H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
"http://127.0.0.1:18080/__memind_canary/route-probe?${probe_query}"
"http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/route-probe?${probe_query}"
)"
printf '%s' "${route_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
let input = "";
@@ -475,7 +542,21 @@ for probe_query in 'username=john' 'wechat_user_id=wx_ul610et8'; do
'
done
say "Drain active Agent runs for the bounded tunnel switch"
say "Start the isolated 105 reverse tunnel on 19082"
write_canary_tunnel_plist
for _ in $(seq 1 30); do
edge_code="$(
edge_ssh \
"curl -sS -o /dev/null -w '%{http_code}' \
--max-time 5 http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status" \
2>/dev/null || true
)"
[[ "${edge_code}" == "200" ]] && break
sleep 2
done
[[ "${edge_code:-}" == "200" ]]
say "Drain active Agent runs for the bounded edge switch"
touch "${STABLE_DIR}/.release-drain"
deadline=$(( $(date +%s) + 120 ))
while (( $(date +%s) < deadline )); do
@@ -504,15 +585,31 @@ while (( $(date +%s) < deadline )); do
done
[[ "${counts:-unknown unknown}" == "0 0" ]]
say "Switch only the reverse tunnel from stable 8081 to identity router 18080"
write_tunnel_plist 18080
sleep 2
remote_code="$(
ssh -o BatchMode=yes -o ConnectTimeout=10 "${MEMIND_PORTAL_TUNNEL_HOST}" \
"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}/api/status" \
2>/dev/null || true
)"
[[ "${remote_code}" == "200" ]]
say "Switch the committed 105 nginx upstreams to the isolated canary tunnel"
edge_ssh /bin/bash <<EDGE_SWITCH
set -euo pipefail
for config in '${EDGE_MOBILE_CONFIG}' '${EDGE_WECHAT_CONFIG}'; do
before_count="\$(grep -cF 'proxy_pass http://58.38.22.103:8081;' "\${config}")"
[[ "\${before_count}" -gt 0 ]]
temp_config="\$(mktemp "\${config}.canary.XXXXXX")"
sed 's#proxy_pass http://58\.38\.22\.103:8081;#proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};#g' \
"\${config}" > "\${temp_config}"
after_count="\$(grep -cF 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' "\${temp_config}")"
[[ "\${after_count}" == "\${before_count}" ]]
chown root:root "\${temp_config}"
chmod 0644 "\${temp_config}"
mv "\${temp_config}" "\${config}"
done
nginx -t
systemctl reload nginx
curl -fsS --max-time 10 http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status >/dev/null
curl -kfsS --max-time 15 \
--resolve m.tkmind.cn:443:127.0.0.1 \
https://m.tkmind.cn/api/status >/dev/null
curl -kfsS --max-time 15 \
--resolve wechat.m.tkmind.cn:443:127.0.0.1 \
https://wechat.m.tkmind.cn/api/status >/dev/null
EDGE_SWITCH
rm -f "${STABLE_DIR}/.release-drain"
say "Verify candidate failure automatically falls back to stable"
@@ -526,7 +623,7 @@ done
fallback_probe="$(
curl -fsS \
-H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
'http://127.0.0.1:18080/__memind_canary/route-probe?username=john'
"http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/route-probe?username=john"
)"
printf '%s' "${fallback_probe}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
let input = "";
@@ -537,6 +634,9 @@ printf '%s' "${fallback_probe}" | /opt/homebrew/opt/node@24/bin/node --input-typ
if (status.candidate !== false || status.route !== "stable") process.exit(1);
});
'
edge_ssh \
"curl -fsS --max-time 10 \
http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT}/api/status >/dev/null"
launchctl bootstrap "${LAUNCHD_GUI}" "${CANDIDATE_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null
for _ in $(seq 1 60); do
@@ -547,7 +647,7 @@ candidate_healthy
final_diagnostic="$(
curl -fsS \
-H "X-Memind-Canary-Secret: ${diagnostic_secret}" \
http://127.0.0.1:18080/__memind_canary/health
"http://127.0.0.1:${CANARY_PROXY_PORT}/__memind_canary/health"
)"
printf '%s' "${final_diagnostic}" | /opt/homebrew/opt/node@24/bin/node --input-type=module -e '
let input = "";
@@ -564,8 +664,11 @@ printf 'release_id=%s\n' "${RELEASE_ID}"
printf 'candidate_dir=%s\n' "${CANDIDATE_DIR}"
printf 'full_backup=%s\n' "${FULL_BACKUP_TAR}"
printf 'persist_backup=%s\n' "${PERSIST_BACKUP_TAR}"
printf 'edge_mobile_backup=%s:%s\n' "${EDGE_HOST}" "${EDGE_MOBILE_BACKUP}"
printf 'edge_wechat_backup=%s:%s\n' "${EDGE_HOST}" "${EDGE_WECHAT_BACKUP}"
printf 'stable_health=http://127.0.0.1:8081/api/status\n'
printf 'proxy_health=http://127.0.0.1:18080/__memind_canary/health\n'
printf 'proxy_health=http://127.0.0.1:%s/__memind_canary/health\n' "${CANARY_PROXY_PORT}"
printf 'edge_tunnel=http://127.0.0.1:%s/api/status\n' "${CANARY_TUNNEL_REMOTE_PORT}"
printf 'candidate_health=http://127.0.0.1:18081/api/status\n'
printf 'candidate_goosed=https://127.0.0.1:18015/status\n'
REMOTE_SCRIPT
scripts/rollback-portal-canary-prod.sh
+40 -41
View File
@@ -5,9 +5,11 @@ HOST="${STUDIO_HOST:-58.38.22.103}"
STABLE_DIR="${STUDIO_REMOTE_ROOT:-/Users/john/Project}/Memind"
PORTAL_CANDIDATE_LABEL="cn.tkmind.memind-portal-candidate"
CANARY_PROXY_LABEL="cn.tkmind.memind-canary-proxy"
PORTAL_TUNNEL_LABEL="cn.tkmind.memind-portal-tunnel"
MEMIND_PORTAL_TUNNEL_HOST="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}"
MEMIND_PORTAL_TUNNEL_REMOTE_PORT="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}"
CANARY_TUNNEL_LABEL="cn.tkmind.memind-canary-tunnel"
EDGE_HOST="${MEMIND_CANARY_EDGE_HOST:-ssh105-public}"
CANARY_TUNNEL_REMOTE_PORT=19082
EDGE_MOBILE_CONFIG="/etc/nginx/conf.d/m.tkmind.cn.conf"
EDGE_WECHAT_CONFIG="/etc/nginx/conf.d/wechat.m.tkmind.cn.conf"
AUTO_YES=0
if [[ "${1:-}" == "--yes" || "${1:-}" == "-y" ]]; then
@@ -18,7 +20,7 @@ elif [[ $# -gt 0 ]]; then
fi
if [[ "${AUTO_YES}" -ne 1 ]]; then
echo "This restores the 105 tunnel to stable Portal 8081 and stops only canary services."
echo "This restores both 105 nginx upstreams to stable Portal 8081 and stops only canary services."
read -r -p "Continue with 103 canary rollback? [y/N] " confirm </dev/tty
[[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
fi
@@ -27,55 +29,52 @@ ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" \
"STABLE_DIR='${STABLE_DIR}' \
PORTAL_CANDIDATE_LABEL='${PORTAL_CANDIDATE_LABEL}' \
CANARY_PROXY_LABEL='${CANARY_PROXY_LABEL}' \
PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' \
MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' \
MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' \
CANARY_TUNNEL_LABEL='${CANARY_TUNNEL_LABEL}' \
EDGE_HOST='${EDGE_HOST}' \
CANARY_TUNNEL_REMOTE_PORT='${CANARY_TUNNEL_REMOTE_PORT}' \
EDGE_MOBILE_CONFIG='${EDGE_MOBILE_CONFIG}' \
EDGE_WECHAT_CONFIG='${EDGE_WECHAT_CONFIG}' \
/bin/bash" <<'REMOTE'
set -euo pipefail
LAUNCHD_GUI="gui/$(id -u)"
TUNNEL_PLIST="${HOME}/Library/LaunchAgents/${PORTAL_TUNNEL_LABEL}.plist"
ssh -o BatchMode=yes -o ConnectTimeout=10 "${EDGE_HOST}" /bin/bash <<EDGE_ROLLBACK
set -euo pipefail
for config in '${EDGE_MOBILE_CONFIG}' '${EDGE_WECHAT_CONFIG}'; do
canary_count="\$(grep -cF 'proxy_pass http://127.0.0.1:${CANARY_TUNNEL_REMOTE_PORT};' "\${config}" || true)"
stable_count_before="\$(grep -cF 'proxy_pass http://58.38.22.103:8081;' "\${config}" || true)"
if [[ "\${canary_count}" -eq 0 ]]; then
[[ "\${stable_count_before}" -gt 0 ]]
continue
fi
temp_config="\$(mktemp "\${config}.stable.XXXXXX")"
sed 's#proxy_pass http://127\.0\.0\.1:${CANARY_TUNNEL_REMOTE_PORT};#proxy_pass http://58.38.22.103:8081;#g' \
"\${config}" > "\${temp_config}"
stable_count="\$(grep -cF 'proxy_pass http://58.38.22.103:8081;' "\${temp_config}")"
[[ "\${stable_count}" -eq \$(( stable_count_before + canary_count )) ]]
chown root:root "\${temp_config}"
chmod 0644 "\${temp_config}"
mv "\${temp_config}" "\${config}"
done
nginx -t
systemctl reload nginx
curl -kfsS --max-time 15 \
--resolve m.tkmind.cn:443:127.0.0.1 \
https://m.tkmind.cn/api/status >/dev/null
curl -kfsS --max-time 15 \
--resolve wechat.m.tkmind.cn:443:127.0.0.1 \
https://wechat.m.tkmind.cn/api/status >/dev/null
EDGE_ROLLBACK
cat > "${TUNNEL_PLIST}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key><string>${PORTAL_TUNNEL_LABEL}</string>
<key>ProgramArguments</key>
<array><string>${STABLE_DIR}/scripts/memind-portal-tunnel.sh</string></array>
<key>EnvironmentVariables</key>
<dict>
<key>MEMIND_PORTAL_TUNNEL_HOST</key><string>${MEMIND_PORTAL_TUNNEL_HOST}</string>
<key>MEMIND_PORTAL_TUNNEL_LOCAL_PORT</key><string>8081</string>
<key>MEMIND_PORTAL_TUNNEL_REMOTE_PORT</key><string>${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}</string>
</dict>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
</dict>
</plist>
EOF
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl bootstrap "${LAUNCHD_GUI}" "${TUNNEL_PLIST}" >/dev/null
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_TUNNEL_LABEL}" >/dev/null
launchctl bootout "${LAUNCHD_GUI}/${CANARY_TUNNEL_LABEL}" >/dev/null 2>&1 || true
launchctl bootout "${LAUNCHD_GUI}/${CANARY_PROXY_LABEL}" >/dev/null 2>&1 || true
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_CANDIDATE_LABEL}" >/dev/null 2>&1 || true
/opt/homebrew/bin/docker rm -f goosed-prod-canary >/dev/null 2>&1 || true
rm -f "${STABLE_DIR}/.release-drain"
curl -fsS http://127.0.0.1:8081/api/status >/dev/null
sleep 2
remote_code="$(
ssh -o BatchMode=yes -o ConnectTimeout=10 "${MEMIND_PORTAL_TUNNEL_HOST}" \
"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}/api/status" \
2>/dev/null || true
)"
[[ "${remote_code}" == "200" ]]
printf 'stable_port=8081\n'
printf 'remote_tunnel_status=%s\n' "${remote_code}"
printf 'edge_upstream=58.38.22.103:8081\n'
REMOTE
echo "103 canary rollback completed; stable Portal remains active on 8081."
scripts/run-memind-canary-proxy-prod.sh
+1 -1
View File
@@ -23,7 +23,7 @@ export NODE_ENV=production
export MEMIND_PORTAL_H5_ROOT="${STABLE_ROOT}"
export MEMIND_CANARY_DIAGNOSTIC_SECRET="$(tr -d '\r\n' < "${SECRET_FILE}")"
export MEMIND_CANARY_PROXY_HOST=127.0.0.1
export MEMIND_CANARY_PROXY_PORT="${MEMIND_CANARY_PROXY_PORT:-18080}"
export MEMIND_CANARY_PROXY_PORT="${MEMIND_CANARY_PROXY_PORT:-18082}"
export MEMIND_CANARY_STABLE_URL="${MEMIND_CANARY_STABLE_URL:-http://127.0.0.1:8081}"
export MEMIND_CANARY_CANDIDATE_URL="${MEMIND_CANARY_CANDIDATE_URL:-http://127.0.0.1:18081}"