Files
memind/scripts/release-portal-runtime-prod.sh

874 lines
32 KiB
Bash
Executable File
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
HOST="${STUDIO_HOST:-58.38.22.103}"
REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}"
APP_DIR="${REMOTE_ROOT}/Memind"
INCOMING_DIR="${REMOTE_ROOT}/incoming/memind-portal-runtime"
BACKUP_DIR="${REMOTE_ROOT}/backups/memind"
ARCHIVE_DIR="${REMOTE_ROOT}/archives"
HEALTH_URL="${STUDIO_HEALTH_URL:-http://127.0.0.1:8081/api/status}"
PORTAL_LABEL="cn.tkmind.memind-portal"
PORTAL_TUNNEL_LABEL="cn.tkmind.memind-portal-tunnel"
MEMIND_PORTAL_TUNNEL_HOST="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}"
MEMIND_PORTAL_TUNNEL_REMOTE_PORT="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}"
LAUNCHD_GUI="gui/$(id -u)"
RELEASE_TS="$(date +%Y%m%d-%H%M%S)"
SHORT_SHA="$(git -C "${ROOT}" rev-parse --short HEAD 2>/dev/null || echo no-git)"
RELEASE_ID="${RELEASE_TS}-${SHORT_SHA}"
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/memind-portal-runtime-release.XXXXXX")"
RUNTIME_ROOT="${ROOT}/.runtime/portal"
BUNDLE_PATH="${TMP_DIR}/memind-portal-runtime-${RELEASE_ID}.tar.gz"
MANIFEST_PATH="${TMP_DIR}/memind-portal-runtime-${RELEASE_ID}.manifest.txt"
SHA_PATH="${TMP_DIR}/memind-portal-runtime-${RELEASE_ID}.sha256"
DRY_RUN=0
SKIP_TESTS=0
SKIP_BUILD=0
AUTO_YES=0
cleanup() {
rm -rf "${TMP_DIR}"
}
trap cleanup EXIT
usage() {
cat <<'EOF'
用法:
bash scripts/release-portal-runtime-prod.sh [--dry-run] [--skip-tests] [--skip-build] [--yes]
说明:
1. 本地构建 Portal 无源码 runtime artifact
2. 上传到 103(固定公网地址)
3. 103 备份当前 Memind 全目录 + 持久目录
4. 停止旧 Portal 服务
5. 用 runtime artifact 替换 live 目录,只继承持久目录
6. 启动 Portal,保持旧端口 8081
7. 健康检查通过后重启 m.tkmind.cn 反向隧道 (105:19081 -> 103:8081)
8. 旧源码目录移入 archive,不再保留可运行 live 源码
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
--dry-run) DRY_RUN=1 ;;
--skip-tests) SKIP_TESTS=1 ;;
--skip-build) SKIP_BUILD=1 ;;
--yes|-y) AUTO_YES=1 ;;
-h|--help)
usage
exit 0
;;
*)
echo "未知参数: $1" >&2
usage >&2
exit 1
;;
esac
shift
done
if [[ "${SKIP_TESTS}" -eq 1 ]]; then
echo "生产发布守门员禁止 --skip-tests。" >&2
exit 1
fi
if [[ "${ALLOW_PORTAL_RELEASE_SCOPE_BYPASS:-0}" == "1" ]]; then
echo "生产发布守门员禁止 ALLOW_PORTAL_RELEASE_SCOPE_BYPASS。" >&2
exit 1
fi
if [[ "${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}" == "1" ]]; then
echo "生产发布守门员禁止 ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES。" >&2
exit 1
fi
if [[ "${DRY_RUN}" -eq 0 ]]; then
say "验证 103 灰度晋升证据"
node "${ROOT}/scripts/verify-canary-promotion-evidence.mjs" --host "${HOST}"
fi
say() {
printf '\n[%s] %s\n' "$(date +%H:%M:%S)" "$*"
}
need_cmd() {
command -v "$1" >/dev/null 2>&1 || {
echo "缺少命令: $1" >&2
exit 1
}
}
need_cmd ssh
need_cmd scp
need_cmd tar
need_cmd shasum
if [[ -x "${ROOT}/scripts/check-release-ready.sh" ]]; then
ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh"
else
echo "缺少发布前闸门: ${ROOT}/scripts/check-release-ready.sh" >&2
exit 1
fi
check_release_scope() {
git -C "${ROOT}" rev-parse --is-inside-work-tree >/dev/null 2>&1 || return 0
local path
local blocked=()
while IFS= read -r path; do
[[ -n "${path}" ]] || continue
case "${path}" in
docker/*|scripts/start-goosed-local.sh|scripts/verify-goosed-docker.sh|docs/goosed-*.md|server.mjs.bak*|*.bak)
blocked+=("${path}")
;;
docs/local-dev.md|README.md)
blocked+=("${path}")
;;
esac
done < <(git -C "${ROOT}" status --short --untracked-files=all | sed -E 's/^.. //')
if [[ "${#blocked[@]}" -gt 0 ]]; then
echo "检测到默认不应并入 103 Portal 发版的本地改动:" >&2
printf ' - %s\n' "${blocked[@]}" >&2
echo "请先清理这些改动,或确认本次需求明确包含这些模块后再重试。" >&2
exit 1
fi
}
say "本地预检查"
check_release_scope
if [[ "${SKIP_TESTS}" -ne 1 ]]; then
say "运行最小验证"
(
cd "${ROOT}"
npm test -- --test-name-pattern='publish|space|billing' >/dev/null
npm run verify:mindspace-publish-guards >/dev/null
npm run verify:page-data >/dev/null
)
fi
if [[ "${SKIP_BUILD}" -ne 1 ]]; then
say "构建 Portal runtime"
(
cd "${ROOT}"
node scripts/build-portal-runtime.mjs
)
fi
[[ -d "${RUNTIME_ROOT}" ]] || {
echo "缺少 runtime 目录: ${RUNTIME_ROOT}" >&2
exit 1
}
verify_runtime_artifact() {
local missing=0
for required in server.mjs memind-canary-proxy.mjs deepseek-no-think-proxy.mjs wechat-mp.bundle.mjs mindspace-sandbox-mcp.mjs tkmind-search-mcp.mjs tkmind-excel-mcp.mjs mindspace-public-links.mjs dist package.json scripts/run-memind-portal-prod.sh scripts/run-memind-portal-candidate.sh scripts/run-memind-canary-proxy-prod.sh scripts/run-deepseek-compat-proxy-candidate.sh scripts/goosed-canary.compose.yml scripts/check-mindspace-public-links.mjs scripts/load-env.mjs scripts/wechat-mp-menu.mjs scripts/memind-portal-tunnel.sh; do
if [[ ! -e "${RUNTIME_ROOT}/${required}" ]]; then
echo "runtime 产物缺失: ${RUNTIME_ROOT}/${required}" >&2
missing=1
fi
done
if [[ "${missing}" -ne 0 ]]; then
echo "请先执行: node scripts/build-portal-runtime.mjs" >&2
exit 1
fi
}
verify_runtime_artifact
say "验证 MindSpace 发布与聊天 Finish 回归守卫"
(
cd "${ROOT}"
npm run verify:mindspace-publish-guards:full
)
verify_mindspace_public_links() {
local target_root="${1:-${ROOT}/MindSpace}"
local label="${2:-本地 MindSpace}"
if [[ ! -d "${target_root}" ]]; then
say "跳过 ${label} 公开页链接检查(目录不存在: ${target_root}"
return 0
fi
say "检查 ${label} 公开页相对链接"
if ! node "${ROOT}/scripts/check-mindspace-public-links.mjs" --root "${target_root}" --downloads-only; then
echo "MindSpace 公开页存在缺失的相对下载/资源链接。" >&2
echo "修复 public/*.html 中的 href/src/cover 路径后再发布。" >&2
exit 1
fi
}
if [[ "${DRY_RUN}" -ne 1 ]]; then
verify_mindspace_public_links "${ROOT}/MindSpace" "本地"
fi
verify_remote_goosed_dependency() {
say "检查 103 goosed 依赖"
ssh -o BatchMode=yes "${HOST}" 'bash -s' <<'REMOTE'
set -euo pipefail
missing=0
pg_isready_bin="/opt/homebrew/opt/postgresql@17/bin/pg_isready"
psql_bin="/opt/homebrew/opt/postgresql@17/bin/psql"
if [[ -x "${pg_isready_bin}" ]]; then
if ! "${pg_isready_bin}" -h 127.0.0.1 -p 5432 -q 2>/dev/null; then
echo "goosed dependency check failed: host PostgreSQL (127.0.0.1:5432) is not accepting connections" >&2
echo "Run: bash /Users/john/Project/goosed-prod/scripts/ensure-goose-session-postgres.sh" >&2
missing=1
fi
else
echo "goosed dependency check warning: pg_isready not found; skipping PostgreSQL readiness check" >&2
fi
portal_env="/Users/john/Project/Memind/.env"
userdata_pg_url=""
if [[ -f "${portal_env}" ]]; then
userdata_pg_url="$(grep -E '^MINDSPACE_USERDATA_PG_URL=' "${portal_env}" | tail -1 | cut -d= -f2- || true)"
fi
if [[ -z "${userdata_pg_url}" ]]; then
echo "goosed dependency check failed: Portal .env must set MINDSPACE_USERDATA_PG_URL" >&2
missing=1
elif [[ -x "${psql_bin}" ]] && ! "${psql_bin}" "${userdata_pg_url}" -Atc 'SELECT 1' >/dev/null 2>&1; then
echo "goosed dependency check failed: user-space PostgreSQL URL cannot execute SELECT 1" >&2
missing=1
fi
docker_bin="/opt/homebrew/bin/docker"
goosed_containers=()
goosed_indexes=()
if [[ -x "${docker_bin}" ]]; then
while IFS='|' read -r name service; do
if [[ "${service}" =~ ^goosed-([0-9]+)$ ]]; then
goosed_containers+=("${name}")
goosed_indexes+=("${BASH_REMATCH[1]}")
fi
done < <("${docker_bin}" ps \
--filter 'label=com.docker.compose.project=goosed-prod' \
--format '{{.Names}}|{{.Label "com.docker.compose.service"}}')
fi
if ((${#goosed_containers[@]} > 0)); then
for position in "${!goosed_containers[@]}"; do
container="${goosed_containers[$position]}"
index="${goosed_indexes[$position]}"
host_port=$((18005 + index))
health="$("${docker_bin}" inspect "${container}" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' 2>/dev/null || true)"
if [[ "${health}" != "healthy" ]] || ! "${docker_bin}" port "${container}" 18006/tcp 2>/dev/null | grep -q "0.0.0.0:${host_port}$"; then
echo "goosed dependency check failed: ${container} is not healthy on host port ${host_port}" >&2
missing=1
fi
if ! "${docker_bin}" exec "${container}" sh -lc 'test -x /usr/local/bin/node && test -f /opt/portal/mindspace-sandbox-mcp.mjs' >/dev/null 2>&1; then
echo "goosed dependency check failed: ${container} missing /usr/local/bin/node or /opt/portal/mindspace-sandbox-mcp.mjs" >&2
missing=1
fi
if ! "${docker_bin}" exec "${container}" /usr/local/bin/node -e '
const net = require("net");
const socket = net.connect({ host: "host.docker.internal", port: 5433 });
const fail = () => process.exit(1);
socket.setTimeout(3000, fail);
socket.once("error", fail);
socket.once("connect", () => { socket.end(); process.exit(0); });
' >/dev/null 2>&1; then
echo "goosed dependency check failed: ${container} cannot reach user-space PostgreSQL at host.docker.internal:5433" >&2
missing=1
fi
done
if [[ -f /Users/john/Project/Memind/.env ]]; then
portal_mcp_node="$(grep -E '^GOOSED_MCP_NODE_PATH=' /Users/john/Project/Memind/.env | tail -1 | cut -d= -f2- || true)"
portal_mcp_server="$(grep -E '^GOOSED_MCP_SERVER_PATH=' /Users/john/Project/Memind/.env | tail -1 | cut -d= -f2- || true)"
if [[ -z "${portal_mcp_node}" || -z "${portal_mcp_server}" ]]; then
echo "goosed dependency check failed: Portal .env must set GOOSED_MCP_NODE_PATH and GOOSED_MCP_SERVER_PATH for Docker goosed" >&2
missing=1
else
for container in "${goosed_containers[@]}"; do
if ! "${docker_bin}" exec "${container}" sh -lc "test -x '${portal_mcp_node}' && test -f '${portal_mcp_server}'" >/dev/null 2>&1; then
echo "goosed dependency check failed: ${container} cannot resolve Portal .env MCP paths '${portal_mcp_node}' and '${portal_mcp_server}'" >&2
missing=1
fi
done
fi
targets_line="$(grep -E '^TKMIND_API_TARGETS=' /Users/john/Project/Memind/.env | tail -1 | cut -d= -f2- || true)"
if [[ -z "${targets_line}" ]]; then
echo "goosed dependency check failed: Portal .env must set TKMIND_API_TARGETS for Docker goosed pool" >&2
missing=1
else
IFS=',' read -ra target_urls <<< "${targets_line}"
for target_url in "${target_urls[@]}"; do
trimmed="$(echo "${target_url}" | tr -d ' \"')"
if [[ -z "${trimmed}" ]]; then
continue
fi
status="$(curl -skS -m 5 "${trimmed}/status" 2>/dev/null || true)"
if [[ "${status}" != "ok" ]]; then
echo "goosed dependency check failed: ${trimmed}/status != ok (${status:-empty})" >&2
missing=1
fi
done
if ((${#target_urls[@]} != ${#goosed_indexes[@]})); then
echo "goosed dependency check warning: TKMIND_API_TARGETS count (${#target_urls[@]}) != running containers (${#goosed_indexes[@]})" >&2
fi
fi
fi
else
api_secret="$(grep -E '^TKMIND_SERVER__SECRET_KEY=' /Users/john/Project/Memind/.env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
for port in 18006 18007; do
status="$(curl -skS -m 5 -H "X-Secret-Key: ${api_secret}" "https://127.0.0.1:${port}/status" 2>/dev/null || true)"
if [[ "${status}" != "ok" ]]; then
line="$(lsof -nP -iTCP:${port} -sTCP:LISTEN 2>/dev/null | awk 'NR==2 {print $1, $2, $9}')"
echo "goosed dependency check failed: port ${port} status check failed (${line:-no listener})" >&2
missing=1
fi
done
portal_mcp_node="$(grep -E '^GOOSED_MCP_NODE_PATH=' /Users/john/Project/Memind/.env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
portal_mcp_server="$(grep -E '^GOOSED_MCP_SERVER_PATH=' /Users/john/Project/Memind/.env 2>/dev/null | tail -1 | cut -d= -f2- || true)"
portal_mcp_node="${portal_mcp_node:-/opt/homebrew/opt/node@24/bin/node}"
portal_mcp_server="${portal_mcp_server:-/Users/john/Project/Memind/mindspace-sandbox-mcp.mjs}"
if [[ ! -x "${portal_mcp_node}" || ! -f "${portal_mcp_server}" ]]; then
echo "goosed dependency check failed: Portal MCP paths are not valid: ${portal_mcp_node}, ${portal_mcp_server}" >&2
missing=1
fi
fi
exit "${missing}"
REMOTE
}
say "验证与当前 main 和 runtime artifact 绑定的 Gate report"
if ! node "${ROOT}/scripts/verify-release-gate-report.mjs" --artifact "${RUNTIME_ROOT}" >/dev/null 2>&1; then
say "执行核心场景 + 变更影响域 Gate"
DEPLOYED_SHA="${MEMIND_RELEASE_BASE_COMMIT:-}"
if [[ -z "${DEPLOYED_SHA}" && "${DRY_RUN}" -ne 1 ]]; then
DEPLOYED_SHA="$(
ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" \
"grep -E '^git_head=' '${APP_DIR}/.release-manifest.txt' 2>/dev/null | tail -1 | cut -d= -f2-" \
2>/dev/null || true
)"
fi
IMPACT_ARGS=(--artifact "${RUNTIME_ROOT}")
if [[ -n "${DEPLOYED_SHA}" ]]; then
IMPACT_ARGS+=(--deployed-commit "${DEPLOYED_SHA}")
fi
node "${ROOT}/scripts/run-release-gate-impact.mjs" "${IMPACT_ARGS[@]}"
fi
node "${ROOT}/scripts/verify-release-gate-report.mjs" --artifact "${RUNTIME_ROOT}"
if [[ "${DRY_RUN}" -ne 1 ]]; then
say "执行 103 只读预检"
ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" "echo portal-runtime-ssh-ok" >/dev/null
ssh -o BatchMode=yes "${HOST}" "test -d '${APP_DIR}' && test -f '${APP_DIR}/.env'" >/dev/null
verify_remote_goosed_dependency
fi
if [[ "${AUTO_YES}" -ne 1 && "${DRY_RUN}" -ne 1 ]]; then
say "发布确认"
echo "目标主机: ${HOST}"
echo "目标目录: ${APP_DIR}"
echo "发布编号: ${RELEASE_ID}"
echo "本地 HEAD: $(git -C "${ROOT}" rev-parse HEAD 2>/dev/null || echo unknown)"
echo "说明: 此次只切换 test-memind Portal 到无源码 runtime,不包含 memindadm / memindplaza"
read -r -p "确认继续发布到 103? [y/N] " confirm </dev/tty
[[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
fi
say "生成发布清单"
{
echo "release_id=${RELEASE_ID}"
echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')"
echo "host=$(hostname)"
echo "git_head=$(git -C "${ROOT}" rev-parse HEAD 2>/dev/null || echo unknown)"
echo "git_branch=$(git -C "${ROOT}" branch --show-current 2>/dev/null || echo detached)"
echo "artifact=.runtime/portal"
echo "persisted_items=.env, MindSpace, data, users, .tailscale, public/plaza-covers, logs"
} > "${MANIFEST_PATH}"
say "打包 Portal runtime artifact"
tar -czf "${BUNDLE_PATH}" -C "${RUNTIME_ROOT}" .
checksum="$(shasum -a 256 "${BUNDLE_PATH}" | awk '{print $1}')"
printf '%s %s\n' "${checksum}" "$(basename "${BUNDLE_PATH}")" > "${SHA_PATH}"
if [[ "${DRY_RUN}" -eq 1 ]]; then
say "dry-run 完成"
ls -lh "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}"
exit 0
fi
say "上传 Portal runtime 到 103"
ssh -o BatchMode=yes "${HOST}" "mkdir -p '${INCOMING_DIR}' '${BACKUP_DIR}' '${ARCHIVE_DIR}'"
scp -q "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}" "${HOST}:${INCOMING_DIR}/"
say "在 103 停旧服务并切换到无源码 runtime"
ssh -o BatchMode=yes "${HOST}" \
"RELEASE_ID='${RELEASE_ID}' APP_DIR='${APP_DIR}' INCOMING_DIR='${INCOMING_DIR}' BACKUP_DIR='${BACKUP_DIR}' ARCHIVE_DIR='${ARCHIVE_DIR}' HEALTH_URL='${HEALTH_URL}' PORTAL_LABEL='${PORTAL_LABEL}' PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' LAUNCHD_GUI='${LAUNCHD_GUI}' ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES='${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}' /bin/bash" <<'REMOTE_SCRIPT'
set -euo pipefail
export PATH="/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin:${PATH}"
RUNTIME_DIR="${APP_DIR}.runtime-${RELEASE_ID}"
OLD_LIVE_DIR="${ARCHIVE_DIR}/Memind-source-before-${RELEASE_ID}"
FAILED_LIVE_DIR="${ARCHIVE_DIR}/Memind-failed-after-${RELEASE_ID}"
BUNDLE="${INCOMING_DIR}/memind-portal-runtime-${RELEASE_ID}.tar.gz"
MANIFEST="${INCOMING_DIR}/memind-portal-runtime-${RELEASE_ID}.manifest.txt"
SHA_FILE="${INCOMING_DIR}/memind-portal-runtime-${RELEASE_ID}.sha256"
remount_goosed_after_live_swap() {
local docker_bin="/opt/homebrew/bin/docker"
local compose_dir="/Users/john/Project/goosed-prod"
local compose_file="${compose_dir}/docker-compose.prod.yml"
local marker_rel="MindSpace/.goosed-remount-${RELEASE_ID}"
local marker_host="${APP_DIR}/${marker_rel}"
local marker_value="${RELEASE_ID}-$(date +%s)"
local ready=0
if [[ ! -x "${docker_bin}" || ! -f "${compose_file}" ]]; then
echo "goosed remount failed: docker or ${compose_file} is unavailable" >&2
return 1
fi
mkdir -p "${APP_DIR}/MindSpace"
printf '%s\n' "${marker_value}" > "${marker_host}"
say "重建 goosed 容器以刷新 Portal/MindSpace bind mount"
if ! (
cd "${compose_dir}"
"${docker_bin}" compose -f "${compose_file}" up -d --force-recreate
); then
rm -f "${marker_host}"
return 1
fi
for _ in $(seq 1 120); do
local healthy=1
local containers=()
while IFS= read -r container; do
[[ -n "${container}" ]] && containers+=("${container}")
done < <("${docker_bin}" ps \
--filter 'label=com.docker.compose.project=goosed-prod' \
--format '{{.Names}}')
if ((${#containers[@]} != 9)); then
healthy=0
fi
local container
for container in "${containers[@]}"; do
local state
state="$("${docker_bin}" inspect "${container}" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' 2>/dev/null || true)"
if [[ "${state}" != "healthy" ]]; then
healthy=0
continue
fi
local observed
observed="$("${docker_bin}" exec "${container}" sh -lc "cat '${APP_DIR}/${marker_rel}'" 2>/dev/null || true)"
if [[ "${observed}" != "${marker_value}" ]]; then
healthy=0
fi
if ! "${docker_bin}" exec "${container}" sh -lc 'test -f /opt/portal/mindspace-sandbox-mcp.mjs' >/dev/null 2>&1; then
healthy=0
fi
done
local port
for port in $(seq 18006 18014); do
if [[ "$(curl -skS -m 5 "https://127.0.0.1:${port}/status" 2>/dev/null || true)" != "ok" ]]; then
healthy=0
fi
done
if [[ "${healthy}" -eq 1 ]]; then
ready=1
break
fi
sleep 2
done
rm -f "${marker_host}"
if [[ "${ready}" -ne 1 ]]; then
echo "goosed remount failed: containers did not become healthy on the new live directory" >&2
return 1
fi
}
FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before.tar.gz"
PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before.tar.gz"
PERSISTED_ITEMS=(
".env"
"MindSpace"
"data"
"users"
".tailscale"
"public/plaza-covers"
"logs"
)
say() {
printf '\n[remote %s] %s\n' "$(date +%H:%M:%S)" "$*"
}
block_new_agent_runs_for_release() {
# The Portal checks this marker before accepting POST /agent/runs. Keep it
# inside the live tree so rollback and cleanup follow the same boundary.
touch "${APP_DIR}/.release-drain"
}
read_agent_run_status_json() {
local worker_script="$1"
node "${worker_script}" --status 2>/dev/null | sed -n '/^{/,$p' || true
}
drain_active_agent_runs() {
local deadline=$(( $(date +%s) + ${MEMIND_RELEASE_DRAIN_TIMEOUT_SEC:-120} ))
local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
while (( $(date +%s) < deadline )); do
if [[ ! -f "${worker_script}" ]]; then
echo "missing agent worker status script: ${worker_script}" >&2
return 1
fi
local status_json
status_json="$(read_agent_run_status_json "${worker_script}")"
if [[ -n "${status_json}" ]]; then
local in_flight pending
in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
pending="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.pendingDispatches||0)); } catch { console.log("unknown"); } });')"
if [[ "${in_flight}" =~ ^[0-9]+$ && "${pending}" =~ ^[0-9]+$ && "${in_flight}" -eq 0 && "${pending}" -eq 0 ]]; then
return 0
fi
fi
sleep 2
done
echo "agent runs did not drain before release timeout" >&2
return 1
}
verify_no_active_agent_runs_before_swap() {
local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
[[ -f "${worker_script}" ]] || return 1
local status_json in_flight pending
status_json="$(read_agent_run_status_json "${worker_script}")"
in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
pending="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.pendingDispatches||0)); } catch { console.log("unknown"); } });')"
[[ "${in_flight}" == "0" && "${pending}" == "0" ]]
}
ensure_portal_tunnel() {
local tunnel_script="${APP_DIR}/scripts/memind-portal-tunnel.sh"
local tunnel_label="${PORTAL_TUNNEL_LABEL:-cn.tkmind.memind-portal-tunnel}"
local tunnel_host="${MEMIND_PORTAL_TUNNEL_HOST:-ssh105-public}"
local remote_port="${MEMIND_PORTAL_TUNNEL_REMOTE_PORT:-19081}"
[[ -x "${tunnel_script}" ]] || {
echo "missing tunnel script: ${tunnel_script}" >&2
return 1
}
cat > "${HOME}/Library/LaunchAgents/${tunnel_label}.plist" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>${tunnel_label}</string>
<key>ProgramArguments</key>
<array>
<string>${tunnel_script}</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>MEMIND_PORTAL_TUNNEL_HOST</key>
<string>${tunnel_host}</string>
</dict>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardOutPath</key>
<string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
<key>StandardErrorPath</key>
<string>${HOME}/Library/Logs/memind-portal-tunnel.log</string>
</dict>
</plist>
EOF
launchctl bootout "${LAUNCHD_GUI}/${tunnel_label}" >/dev/null 2>&1 || true
launchctl bootstrap "${LAUNCHD_GUI}" "${HOME}/Library/LaunchAgents/${tunnel_label}.plist" >/dev/null 2>&1 || true
launchctl enable "${LAUNCHD_GUI}/${tunnel_label}" >/dev/null 2>&1 || true
launchctl kickstart -k "${LAUNCHD_GUI}/${tunnel_label}" >/dev/null 2>&1 || true
sleep 2
if ! launchctl print "${LAUNCHD_GUI}/${tunnel_label}" 2>/dev/null | grep -q 'state = running'; then
echo "portal tunnel failed to start (${tunnel_label})" >&2
return 1
fi
if command -v ssh >/dev/null 2>&1; then
tunnel_code="$(ssh -o BatchMode=yes -o ConnectTimeout=10 "${tunnel_host}" \
"curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:${remote_port}/api/status" 2>/dev/null || true)"
if [[ "${tunnel_code}" == "200" ]]; then
say "105 隧道端口 ${remote_port} 健康"
else
echo "warn: 105:${remote_port} 健康检查未通过 (code=${tunnel_code:-000});请确认 105 nginx upstream 指向 127.0.0.1:${remote_port}" >&2
fi
fi
}
rollback() {
# If the candidate was already swapped in, preserve it before restoring the
# old live. This makes a failed release auditable and keeps rollback
# recoverable instead of silently deleting the candidate.
if [[ -d "${OLD_LIVE_DIR}" && -d "${APP_DIR}" ]]; then
rm -rf "${FAILED_LIVE_DIR}"
mv "${APP_DIR}" "${FAILED_LIVE_DIR}"
mv "${OLD_LIVE_DIR}" "${APP_DIR}"
elif [[ -d "${OLD_LIVE_DIR}" && ! -d "${APP_DIR}" ]]; then
mv "${OLD_LIVE_DIR}" "${APP_DIR}"
fi
launchctl bootstrap "${LAUNCHD_GUI}" "${HOME}/Library/LaunchAgents/${PORTAL_LABEL}.plist" >/dev/null 2>&1 || true
launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_LABEL}" >/dev/null 2>&1 || true
if ! curl -sf "${HEALTH_URL}" >/dev/null 2>&1; then
nohup "${APP_DIR}/scripts/run-memind-portal-prod.sh" >> "${HOME}/Library/Logs/memind-portal.log" 2>&1 &
fi
rm -f "${APP_DIR}/.release-drain"
ensure_portal_tunnel >/dev/null 2>&1 || true
}
trap 'rollback' ERR
[[ -f "${BUNDLE}" ]] || { echo "missing bundle: ${BUNDLE}" >&2; exit 1; }
[[ -f "${MANIFEST}" ]] || { echo "missing manifest: ${MANIFEST}" >&2; exit 1; }
[[ -f "${SHA_FILE}" ]] || { echo "missing sha file: ${SHA_FILE}" >&2; exit 1; }
say "校验产物完整性"
cd "${INCOMING_DIR}"
shasum -a 256 -c "$(basename "${SHA_FILE}")"
copy_persisted_item() {
local src="$1"
local dest_root="$2"
local item="$3"
mkdir -p "${dest_root}"
if [[ -d "${src}" ]]; then
local pipe_status tar_status extract_status
set +e
(
cd "${APP_DIR}"
COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - "${item}"
) | (
cd "${dest_root}"
COPYFILE_DISABLE=1 tar -xf -
)
pipe_status=("${PIPESTATUS[@]}")
tar_status=${pipe_status[0]:-0}
extract_status=${pipe_status[1]:-0}
set -e
if [[ "${extract_status}" -ne 0 ]]; then
echo "failed to extract persisted item ${item}" >&2
exit "${extract_status}"
fi
if [[ "${tar_status}" -ne 0 ]]; then
echo "warning: persisted item ${item} changed during copy; continuing with partial snapshot" >&2
fi
else
cp -a "${src}" "${dest_root}/"
fi
}
say "备份当前 live 全目录"
set +e
COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
-czf "${FULL_BACKUP_TAR}" -C "$(dirname "${APP_DIR}")" "$(basename "${APP_DIR}")"
tar_status=$?
set -e
if [[ ! -s "${FULL_BACKUP_TAR}" ]]; then
echo "full backup failed: ${FULL_BACKUP_TAR}" >&2
exit 1
fi
if [[ "${tar_status}" -ne 0 ]]; then
echo "warning: full backup tar reported errors; continuing with partial archive" >&2
fi
say "单独备份持久目录"
tmp_persist_dir="$(mktemp -d "${TMPDIR:-/tmp}/memind-persisted.XXXXXX")"
for item in "${PERSISTED_ITEMS[@]}"; do
if [[ -e "${APP_DIR}/${item}" ]]; then
mkdir -p "${tmp_persist_dir}/$(dirname "${item}")"
if [[ "${item}" == ".tailscale" && -d "${APP_DIR}/${item}" ]]; then
mkdir -p "${tmp_persist_dir}/${item}"
(
cd "${APP_DIR}/${item}"
COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
) | (
cd "${tmp_persist_dir}/${item}"
tar -xf -
)
else
copy_persisted_item "${APP_DIR}/${item}" "${tmp_persist_dir}" "${item}"
fi
fi
done
COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${tmp_persist_dir}" .
rm -rf "${tmp_persist_dir}"
say "阻止新 Agent Run 并排水现有任务"
block_new_agent_runs_for_release
drain_active_agent_runs
verify_no_active_agent_runs_before_swap
say "停止旧 Portal 服务"
launchctl bootout "${LAUNCHD_GUI}/${PORTAL_LABEL}" >/dev/null 2>&1 || true
lsof -tiTCP:8081 -sTCP:LISTEN 2>/dev/null | xargs kill 2>/dev/null || true
sleep 2
lsof -tiTCP:8081 -sTCP:LISTEN 2>/dev/null | xargs kill -9 2>/dev/null || true
say "准备新的 runtime live 目录"
rm -rf "${RUNTIME_DIR}"
mkdir -p "${RUNTIME_DIR}"
tar -xzf "${BUNDLE}" -C "${RUNTIME_DIR}"
cp "${MANIFEST}" "${RUNTIME_DIR}/.release-manifest.txt"
for item in "${PERSISTED_ITEMS[@]}"; do
if [[ -e "${APP_DIR}/${item}" ]]; then
rm -rf "${RUNTIME_DIR:?}/${item}"
mkdir -p "$(dirname "${RUNTIME_DIR}/${item}")"
if [[ "${item}" == ".tailscale" && -d "${APP_DIR}/${item}" ]]; then
mkdir -p "${RUNTIME_DIR}/${item}"
(
cd "${APP_DIR}/${item}"
COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
) | (
cd "${RUNTIME_DIR}/${item}"
tar -xf -
)
else
copy_persisted_item "${APP_DIR}/${item}" "${RUNTIME_DIR}" "${item}"
fi
fi
done
touch "${RUNTIME_DIR}/.release-drain"
say "将旧源码 live 目录移入 archive"
rm -rf "${OLD_LIVE_DIR}"
mv "${APP_DIR}" "${OLD_LIVE_DIR}"
mv "${RUNTIME_DIR}" "${APP_DIR}"
remount_goosed_after_live_swap
say "更新 LaunchAgent 指向 runtime 启动脚本"
cat > "${HOME}/Library/LaunchAgents/${PORTAL_LABEL}.plist" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>${PORTAL_LABEL}</string>
<key>ProgramArguments</key>
<array>
<string>${APP_DIR}/scripts/run-memind-portal-prod.sh</string>
</array>
<key>WorkingDirectory</key>
<string>${APP_DIR}</string>
<key>EnvironmentVariables</key>
<dict>
<key>PATH</key>
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
</dict>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>ThrottleInterval</key>
<integer>10</integer>
<key>StandardOutPath</key>
<string>${HOME}/Library/Logs/memind-portal.log</string>
<key>StandardErrorPath</key>
<string>${HOME}/Library/Logs/memind-portal.log</string>
</dict>
</plist>
EOF
say "启动新的 Portal runtime"
launchctl bootstrap "${LAUNCHD_GUI}" "${HOME}/Library/LaunchAgents/${PORTAL_LABEL}.plist" >/dev/null 2>&1 || true
launchctl enable "${LAUNCHD_GUI}/${PORTAL_LABEL}" >/dev/null 2>&1 || true
if ! launchctl kickstart -k "${LAUNCHD_GUI}/${PORTAL_LABEL}" >/dev/null 2>&1; then
nohup "${APP_DIR}/scripts/run-memind-portal-prod.sh" >> "${HOME}/Library/Logs/memind-portal.log" 2>&1 &
fi
AGENT_RUN_WORKER_LABEL="${MEMIND_AGENT_RUN_WORKER_LABEL:-cn.tkmind.memind-agent-run-worker}"
if launchctl print "${LAUNCHD_GUI}/${AGENT_RUN_WORKER_LABEL}" >/dev/null 2>&1; then
say "重启 agent run worker"
launchctl kickstart -k "${LAUNCHD_GUI}/${AGENT_RUN_WORKER_LABEL}" >/dev/null 2>&1 || true
fi
say "健康检查"
for _ in $(seq 1 60); do
portal_code="$(curl -s -o /dev/null -w '%{http_code}' "${HEALTH_URL}" || true)"
if [[ "${portal_code}" == "200" ]]; then
break
fi
sleep 2
done
portal_code="$(curl -s -o /dev/null -w '%{http_code}' "${HEALTH_URL}" || true)"
if [[ "${portal_code}" != "200" ]]; then
echo "health check failed: portal=${portal_code}" >&2
exit 1
fi
rm -f "${APP_DIR}/.release-drain"
if [[ "${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}" != "1" && -d "${APP_DIR}/MindSpace" && -f "${APP_DIR}/scripts/check-mindspace-public-links.mjs" ]]; then
say "修复 MindSpace 公开页缺失 docx 下载"
node_bin="/opt/homebrew/opt/node@24/bin/node"
if [[ ! -x "${node_bin}" ]]; then
node_bin="$(command -v node)"
fi
if [[ -f "${APP_DIR}/scripts/repair-mindspace-public-downloads.mjs" ]]; then
"${node_bin}" "${APP_DIR}/scripts/repair-mindspace-public-downloads.mjs" --root "${APP_DIR}/MindSpace" || true
fi
say "检查 MindSpace 公开页相对链接"
if ! "${node_bin}" "${APP_DIR}/scripts/check-mindspace-public-links.mjs" --root "${APP_DIR}/MindSpace" --downloads-only; then
echo "MindSpace public link check failed: broken relative download/asset links under public/*.html" >&2
echo "Set ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES=1 only if you accept shipping with known broken links." >&2
exit 1
fi
fi
say "重启 m.tkmind.cn 反向隧道"
ensure_portal_tunnel
say "检查 live 目录中不再保留源码树"
allowed_live_mjs=(
"${APP_DIR}/server.mjs"
"${APP_DIR}/mindspace-sandbox-mcp.mjs"
"${APP_DIR}/tkmind-search-mcp.mjs"
"${APP_DIR}/tkmind-excel-mcp.mjs"
"${APP_DIR}/mindspace-public-links.mjs"
)
extra_files=""
while IFS= read -r file; do
[[ -z "${file}" ]] && continue
allowed=0
for allowed_file in "${allowed_live_mjs[@]}"; do
if [[ "${file}" == "${allowed_file}" ]]; then
allowed=1
break
fi
done
if [[ "${allowed}" -eq 0 ]]; then
extra_files+="${file} "
fi
done < <(find "${APP_DIR}" -maxdepth 1 \( -name '*.mjs' -o -name 'docs' -o -name 'src' -o -name 'ops' \))
if [[ -n "${extra_files// /}" ]]; then
echo "unexpected source-like files remain in live dir: ${extra_files}" >&2
exit 1
fi
say "Portal runtime 发布完成"
printf 'live_dir=%s\n' "${APP_DIR}"
printf 'archived_source=%s\n' "${OLD_LIVE_DIR}"
printf 'full_backup=%s\n' "${FULL_BACKUP_TAR}"
printf 'persist_backup=%s\n' "${PERSIST_BACKUP_TAR}"
REMOTE_SCRIPT
say "读取 103 最终状态"
ssh -o BatchMode=yes "${HOST}" "curl -s '${HEALTH_URL}' && echo && ls -la '${APP_DIR}' | sed -n '1,80p' && echo '---' && ls -la '${APP_DIR}/scripts' | sed -n '1,40p'"