From 41bf775c4c36885b8710b0bae2f21b5b1459e27d Mon Sep 17 00:00:00 2001 From: john Date: Mon, 27 Jul 2026 07:09:27 +0800 Subject: [PATCH] fix(release): harden stable promotion after canary acceptance Verify matching 103 canary evidence before 8081 promotion, fix agent-run drain JSON parsing, tolerate macOS full-backup tar races, and extend goosed remount health waits for non-interactive SSH releases. Co-authored-by: Cursor --- release-gate/release-script.test.mjs | 15 +++ scripts/release-portal-runtime-prod.sh | 28 +++- scripts/verify-canary-promotion-evidence.mjs | 129 +++++++++++++++++++ 3 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 scripts/verify-canary-promotion-evidence.mjs diff --git a/release-gate/release-script.test.mjs b/release-gate/release-script.test.mjs index c0b12b8..f8899a0 100644 --- a/release-gate/release-script.test.mjs +++ b/release-gate/release-script.test.mjs @@ -42,6 +42,21 @@ test('production release verifies gate report before any 103 connection', async assert.match(source, /禁止 ALLOW_PORTAL_RELEASE_SCOPE_BYPASS/); }); +test('production stable release verifies canary promotion evidence before gate checks', async () => { + const source = await fs.readFile( + path.join(ROOT, 'scripts', 'release-portal-runtime-prod.sh'), + 'utf8', + ); + const promotionIndex = source.indexOf('verify-canary-promotion-evidence.mjs'); + const gateIndex = source.indexOf('verify-release-gate-report.mjs'); + assert.ok(promotionIndex > 0, 'missing canary promotion evidence verifier'); + assert.ok(gateIndex > promotionIndex, 'gate verification must follow promotion evidence'); + assert.doesNotMatch(source, /在同一候选完成 103 灰度验收且晋升证据校验落地前，禁止非 dry-run/); + assert.match(source, /read_agent_run_status_json/); + assert.match(source, /sed -n '\/\^\{/); + assert.match(source, /for _ in \$\(seq 1 120\)/); +}); + test('runtime builder removes every persisted path forbidden by artifact policy', async () => { const source = await fs.readFile( path.join(ROOT, 'scripts', 'build-portal-runtime.mjs'), diff --git a/scripts/release-portal-runtime-prod.sh b/scripts/release-portal-runtime-prod.sh index 5ef95f8..20e578c 100755 --- a/scripts/release-portal-runtime-prod.sh +++ b/scripts/release-portal-runtime-prod.sh @@ -84,10 +84,8 @@ if [[ "${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}" == "1" ]]; then fi if [[ "${DRY_RUN}" -eq 0 ]]; then - echo "当前脚本执行 8081 整包晋升，不是用户级灰度入口。" >&2 - echo "在同一候选完成 103 灰度验收且晋升证据校验落地前，禁止非 dry-run。" >&2 - echo "首次生产动作请使用 scripts/release-portal-canary-prod.sh。" >&2 - exit 1 + say "验证 103 灰度晋升证据" + node "${ROOT}/scripts/verify-canary-promotion-evidence.mjs" --host "${HOST}" fi say() { @@ -388,6 +386,7 @@ say "在 103 停旧服务并切换到无源码 runtime" ssh -o BatchMode=yes "${HOST}" \ "RELEASE_ID='${RELEASE_ID}' APP_DIR='${APP_DIR}' INCOMING_DIR='${INCOMING_DIR}' BACKUP_DIR='${BACKUP_DIR}' ARCHIVE_DIR='${ARCHIVE_DIR}' HEALTH_URL='${HEALTH_URL}' PORTAL_LABEL='${PORTAL_LABEL}' PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' LAUNCHD_GUI='${LAUNCHD_GUI}' ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES='${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}' /bin/bash" <<'REMOTE_SCRIPT' set -euo pipefail +export PATH="/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin:${PATH}" RUNTIME_DIR="${APP_DIR}.runtime-${RELEASE_ID}" OLD_LIVE_DIR="${ARCHIVE_DIR}/Memind-source-before-${RELEASE_ID}" @@ -422,7 +421,7 @@ remount_goosed_after_live_swap() { return 1 fi - for _ in $(seq 1 60); do + for _ in $(seq 1 120); do local healthy=1 local containers=() while IFS= read -r container; do @@ -495,6 +494,11 @@ block_new_agent_runs_for_release() { touch "${APP_DIR}/.release-drain" } +read_agent_run_status_json() { + local worker_script="$1" + node "${worker_script}" --status 2>/dev/null | sed -n '/^{/,$p' || true +} + drain_active_agent_runs() { local deadline=$(( $(date +%s) + ${MEMIND_RELEASE_DRAIN_TIMEOUT_SEC:-120} )) local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs" @@ -504,7 +508,7 @@ drain_active_agent_runs() { return 1 fi local status_json - status_json="$(node "${worker_script}" --status 2>/dev/null || true)" + status_json="$(read_agent_run_status_json "${worker_script}")" if [[ -n "${status_json}" ]]; then local in_flight pending in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')" @@ -523,7 +527,7 @@ verify_no_active_agent_runs_before_swap() { local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs" [[ -f "${worker_script}" ]] || return 1 local status_json in_flight pending - status_json="$(node "${worker_script}" --status 2>/dev/null || true)" + status_json="$(read_agent_run_status_json "${worker_script}")" in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')" pending="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.pendingDispatches||0)); } catch { console.log("unknown"); } });')" [[ "${in_flight}" == "0" && "${pending}" == "0" ]] @@ -652,8 +656,18 @@ copy_persisted_item() { } say "备份当前 live 全目录" +set +e COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \ -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${APP_DIR}")" "$(basename "${APP_DIR}")" +tar_status=$? +set -e +if [[ ! -s "${FULL_BACKUP_TAR}" ]]; then + echo "full backup failed: ${FULL_BACKUP_TAR}" >&2 + exit 1 +fi +if [[ "${tar_status}" -ne 0 ]]; then + echo "warning: full backup tar reported errors; continuing with partial archive" >&2 +fi say "单独备份持久目录" tmp_persist_dir="$(mktemp -d "${TMPDIR:-/tmp}/memind-persisted.XXXXXX")" diff --git a/scripts/verify-canary-promotion-evidence.mjs b/scripts/verify-canary-promotion-evidence.mjs new file mode 100644 index 0000000..a292917 --- /dev/null +++ b/scripts/verify-canary-promotion-evidence.mjs @@ -0,0 +1,129 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import path from 'node:path'; + +const ROOT = path.resolve(new URL('..', import.meta.url).pathname); +const REMOTE_ROOT = process.env.STUDIO_REMOTE_ROOT ?? '/Users/john/Project'; +const CANDIDATE_BASE = `${REMOTE_ROOT}/Memind-candidates`; + +const REMOTE_SCRIPT = [ + 'set -euo pipefail', + 'expected_commit="$1"', + 'candidate_base="$2"', + 'matched=""', + 'latest=""', + 'for dir in "${candidate_base}"/*; do', + ' [[ -d "${dir}" ]] || continue', + ' [[ -f "${dir}/.release-manifest.txt" ]] || continue', + ' if [[ -z "${latest}" || "$(basename "${dir}")" > "$(basename "${latest}")" ]]; then', + ' latest="${dir}"', + ' fi', + ' if grep -q "^git_head=${expected_commit}$" "${dir}/.release-manifest.txt"; then', + ' matched="${dir}"', + ' fi', + 'done', + 'if [[ -n "${matched}" ]]; then candidate="${matched}"; else candidate="${latest}"; fi', + '[[ -n "${candidate}" ]] || { echo "NO_CANDIDATE"; exit 1; }', + 'printf \'candidate_dir=%s\\n\' "${candidate}"', + 'cat "${candidate}/.release-manifest.txt"', + 'printf \'\\n---HEALTH---\\n\'', + 'curl -s -D - -o /dev/null http://127.0.0.1:18081/api/status | tr -d \'\\r\'', + '', +].join('\n'); + +function parseArgs(argv) { + const options = { + host: process.env.STUDIO_HOST ?? '58.38.22.103', + commit: null, + }; + for (let index = 2; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--host') options.host = argv[++index] ?? ''; + else if (arg === '--commit') options.commit = argv[++index] ?? ''; + else throw new Error(`Unknown argument: ${arg}`); + } + return options; +} + +function git(...args) { + const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`); + } + return result.stdout.trim(); +} + +function ssh(host, args, input = '') { + const result = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=15', host, ...args], { + encoding: 'utf8', + input, + }); + if (result.status !== 0) { + throw new Error(`SSH to ${host} failed:\n${result.stderr || result.stdout}`); + } + return result.stdout.trim(); +} + +function parseManifest(content) { + const fields = {}; + for (const line of content.split('\n')) { + const separator = line.indexOf('='); + if (separator <= 0) continue; + fields[line.slice(0, separator)] = line.slice(separator + 1); + } + return fields; +} + +try { + const options = parseArgs(process.argv); + const commit = options.commit ?? git('rev-parse', 'HEAD'); + const remoteUserHost = options.host.includes('@') ? options.host : `john@${options.host}`; + + const remoteOutput = ssh( + remoteUserHost, + ['/bin/bash', '-s', '--', commit, CANDIDATE_BASE], + REMOTE_SCRIPT, + ); + + const healthMarker = '\n---HEALTH---\n'; + const markerIndex = remoteOutput.indexOf('---HEALTH---'); + if (markerIndex < 0) { + throw new Error('Canary promotion evidence response missing health probe'); + } + const manifestText = remoteOutput.slice(0, markerIndex); + const healthText = remoteOutput.slice(markerIndex + healthMarker.trim().length); + + const candidateDir = manifestText + .split('\n') + .find((line) => line.startsWith('candidate_dir=')) + ?.slice('candidate_dir='.length); + const manifest = parseManifest( + manifestText + .split('\n') + .filter((line) => !line.startsWith('candidate_dir=')) + .join('\n'), + ); + + if (!candidateDir) { + throw new Error('103 has no canary candidate directory'); + } + if (manifest.git_head !== commit) { + throw new Error( + `Canary candidate commit mismatch: candidate=${manifest.git_head ?? 'missing'} expected=${commit}`, + ); + } + if (!/^HTTP\/1\.1 200/m.test(healthText)) { + throw new Error('Canary candidate /api/status is not healthy on 18081'); + } + if (!/^x-memind-runtime-role: candidate/im.test(healthText)) { + throw new Error('Canary candidate health probe missing X-Memind-Runtime-Role: candidate'); + } + + console.log('Canary promotion evidence verified'); + console.log(`candidate_dir=${candidateDir}`); + console.log(`release_id=${manifest.release_id ?? 'unknown'}`); + console.log(`git_head=${manifest.git_head}`); +} catch (error) { + console.error(error.message); + process.exit(1); +}