Merge pull request 'fix(release): harden stable promotion after canary acceptance' (#35) from codex/fix-stable-promotion-release into main

Verify matching 103 canary evidence before 8081 promotion, fix agent-run
drain JSON parsing, tolerate macOS full-backup tar races, and extend goosed
remount health waits for non-interactive SSH releases.

Co-authored-by: Cursor <cursoragent@cursor.com>

release-gate/release-script.test.mjs

@@ -42,6 +42,21 @@ test('production release verifies gate report before any 103 connection', async
   assert.match(source, /禁止 ALLOW_PORTAL_RELEASE_SCOPE_BYPASS/);
 });
 
+test('production stable release verifies canary promotion evidence before gate checks', async () => {
+  const source = await fs.readFile(
+    path.join(ROOT, 'scripts', 'release-portal-runtime-prod.sh'),
+    'utf8',
+  );
+  const promotionIndex = source.indexOf('verify-canary-promotion-evidence.mjs');
+  const gateIndex = source.indexOf('verify-release-gate-report.mjs');
+  assert.ok(promotionIndex > 0, 'missing canary promotion evidence verifier');
+  assert.ok(gateIndex > promotionIndex, 'gate verification must follow promotion evidence');
+  assert.doesNotMatch(source, /在同一候选完成 103 灰度验收且晋升证据校验落地前，禁止非 dry-run/);
+  assert.match(source, /read_agent_run_status_json/);
+  assert.match(source, /sed -n '\/\^\{/);
+  assert.match(source, /for _ in \$\(seq 1 120\)/);
+});
+
 test('runtime builder removes every persisted path forbidden by artifact policy', async () => {
   const source = await fs.readFile(
     path.join(ROOT, 'scripts', 'build-portal-runtime.mjs'),

scripts/release-portal-runtime-prod.sh

@@ -84,10 +84,8 @@ if [[ "${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}" == "1" ]]; then
 fi
 
 if [[ "${DRY_RUN}" -eq 0 ]]; then
-  echo "当前脚本执行 8081 整包晋升，不是用户级灰度入口。" >&2
-  echo "在同一候选完成 103 灰度验收且晋升证据校验落地前，禁止非 dry-run。" >&2
-  echo "首次生产动作请使用 scripts/release-portal-canary-prod.sh。" >&2
-  exit 1
+  say "验证 103 灰度晋升证据"
+  node "${ROOT}/scripts/verify-canary-promotion-evidence.mjs" --host "${HOST}"
 fi
 
 say() {
@@ -388,6 +386,7 @@ say "在 103 停旧服务并切换到无源码 runtime"
 ssh -o BatchMode=yes "${HOST}" \
   "RELEASE_ID='${RELEASE_ID}' APP_DIR='${APP_DIR}' INCOMING_DIR='${INCOMING_DIR}' BACKUP_DIR='${BACKUP_DIR}' ARCHIVE_DIR='${ARCHIVE_DIR}' HEALTH_URL='${HEALTH_URL}' PORTAL_LABEL='${PORTAL_LABEL}' PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' LAUNCHD_GUI='${LAUNCHD_GUI}' ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES='${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}' /bin/bash" <<'REMOTE_SCRIPT'
 set -euo pipefail
+export PATH="/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin:${PATH}"
 
 RUNTIME_DIR="${APP_DIR}.runtime-${RELEASE_ID}"
 OLD_LIVE_DIR="${ARCHIVE_DIR}/Memind-source-before-${RELEASE_ID}"
@@ -422,7 +421,7 @@ remount_goosed_after_live_swap() {
     return 1
   fi
 
-  for _ in $(seq 1 60); do
+  for _ in $(seq 1 120); do
     local healthy=1
     local containers=()
     while IFS= read -r container; do
@@ -495,6 +494,11 @@ block_new_agent_runs_for_release() {
   touch "${APP_DIR}/.release-drain"
 }
 
+read_agent_run_status_json() {
+  local worker_script="$1"
+  node "${worker_script}" --status 2>/dev/null | sed -n '/^{/,$p' || true
+}
+
 drain_active_agent_runs() {
   local deadline=$(( $(date +%s) + ${MEMIND_RELEASE_DRAIN_TIMEOUT_SEC:-120} ))
   local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
@@ -504,7 +508,7 @@ drain_active_agent_runs() {
       return 1
     fi
     local status_json
-    status_json="$(node "${worker_script}" --status 2>/dev/null || true)"
+    status_json="$(read_agent_run_status_json "${worker_script}")"
     if [[ -n "${status_json}" ]]; then
       local in_flight pending
       in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
@@ -523,7 +527,7 @@ verify_no_active_agent_runs_before_swap() {
   local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
   [[ -f "${worker_script}" ]] || return 1
   local status_json in_flight pending
-  status_json="$(node "${worker_script}" --status 2>/dev/null || true)"
+  status_json="$(read_agent_run_status_json "${worker_script}")"
   in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
   pending="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.pendingDispatches||0)); } catch { console.log("unknown"); } });')"
   [[ "${in_flight}" == "0" && "${pending}" == "0" ]]
@@ -652,8 +656,18 @@ copy_persisted_item() {
 }
 
 say "备份当前 live 全目录"
+set +e
 COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
   -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${APP_DIR}")" "$(basename "${APP_DIR}")"
+tar_status=$?
+set -e
+if [[ ! -s "${FULL_BACKUP_TAR}" ]]; then
+  echo "full backup failed: ${FULL_BACKUP_TAR}" >&2
+  exit 1
+fi
+if [[ "${tar_status}" -ne 0 ]]; then
+  echo "warning: full backup tar reported errors; continuing with partial archive" >&2
+fi
 
 say "单独备份持久目录"
 tmp_persist_dir="$(mktemp -d "${TMPDIR:-/tmp}/memind-persisted.XXXXXX")"

scripts/verify-canary-promotion-evidence.mjs

@@ -0,0 +1,129 @@
+#!/usr/bin/env node
+import { spawnSync } from 'node:child_process';
+import path from 'node:path';
+
+const ROOT = path.resolve(new URL('..', import.meta.url).pathname);
+const REMOTE_ROOT = process.env.STUDIO_REMOTE_ROOT ?? '/Users/john/Project';
+const CANDIDATE_BASE = `${REMOTE_ROOT}/Memind-candidates`;
+
+const REMOTE_SCRIPT = [
+  'set -euo pipefail',
+  'expected_commit="$1"',
+  'candidate_base="$2"',
+  'matched=""',
+  'latest=""',
+  'for dir in "${candidate_base}"/*; do',
+  '  [[ -d "${dir}" ]] || continue',
+  '  [[ -f "${dir}/.release-manifest.txt" ]] || continue',
+  '  if [[ -z "${latest}" || "$(basename "${dir}")" > "$(basename "${latest}")" ]]; then',
+  '    latest="${dir}"',
+  '  fi',
+  '  if grep -q "^git_head=${expected_commit}$" "${dir}/.release-manifest.txt"; then',
+  '    matched="${dir}"',
+  '  fi',
+  'done',
+  'if [[ -n "${matched}" ]]; then candidate="${matched}"; else candidate="${latest}"; fi',
+  '[[ -n "${candidate}" ]] || { echo "NO_CANDIDATE"; exit 1; }',
+  'printf \'candidate_dir=%s\\n\' "${candidate}"',
+  'cat "${candidate}/.release-manifest.txt"',
+  'printf \'\\n---HEALTH---\\n\'',
+  'curl -s -D - -o /dev/null http://127.0.0.1:18081/api/status | tr -d \'\\r\'',
+  '',
+].join('\n');
+
+function parseArgs(argv) {
+  const options = {
+    host: process.env.STUDIO_HOST ?? '58.38.22.103',
+    commit: null,
+  };
+  for (let index = 2; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--host') options.host = argv[++index] ?? '';
+    else if (arg === '--commit') options.commit = argv[++index] ?? '';
+    else throw new Error(`Unknown argument: ${arg}`);
+  }
+  return options;
+}
+
+function git(...args) {
+  const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8' });
+  if (result.status !== 0) {
+    throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`);
+  }
+  return result.stdout.trim();
+}
+
+function ssh(host, args, input = '') {
+  const result = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=15', host, ...args], {
+    encoding: 'utf8',
+    input,
+  });
+  if (result.status !== 0) {
+    throw new Error(`SSH to ${host} failed:\n${result.stderr || result.stdout}`);
+  }
+  return result.stdout.trim();
+}
+
+function parseManifest(content) {
+  const fields = {};
+  for (const line of content.split('\n')) {
+    const separator = line.indexOf('=');
+    if (separator <= 0) continue;
+    fields[line.slice(0, separator)] = line.slice(separator + 1);
+  }
+  return fields;
+}
+
+try {
+  const options = parseArgs(process.argv);
+  const commit = options.commit ?? git('rev-parse', 'HEAD');
+  const remoteUserHost = options.host.includes('@') ? options.host : `john@${options.host}`;
+
+  const remoteOutput = ssh(
+    remoteUserHost,
+    ['/bin/bash', '-s', '--', commit, CANDIDATE_BASE],
+    REMOTE_SCRIPT,
+  );
+
+  const healthMarker = '\n---HEALTH---\n';
+  const markerIndex = remoteOutput.indexOf('---HEALTH---');
+  if (markerIndex < 0) {
+    throw new Error('Canary promotion evidence response missing health probe');
+  }
+  const manifestText = remoteOutput.slice(0, markerIndex);
+  const healthText = remoteOutput.slice(markerIndex + healthMarker.trim().length);
+
+  const candidateDir = manifestText
+    .split('\n')
+    .find((line) => line.startsWith('candidate_dir='))
+    ?.slice('candidate_dir='.length);
+  const manifest = parseManifest(
+    manifestText
+      .split('\n')
+      .filter((line) => !line.startsWith('candidate_dir='))
+      .join('\n'),
+  );
+
+  if (!candidateDir) {
+    throw new Error('103 has no canary candidate directory');
+  }
+  if (manifest.git_head !== commit) {
+    throw new Error(
+      `Canary candidate commit mismatch: candidate=${manifest.git_head ?? 'missing'} expected=${commit}`,
+    );
+  }
+  if (!/^HTTP\/1\.1 200/m.test(healthText)) {
+    throw new Error('Canary candidate /api/status is not healthy on 18081');
+  }
+  if (!/^x-memind-runtime-role: candidate/im.test(healthText)) {
+    throw new Error('Canary candidate health probe missing X-Memind-Runtime-Role: candidate');
+  }
+
+  console.log('Canary promotion evidence verified');
+  console.log(`candidate_dir=${candidateDir}`);
+  console.log(`release_id=${manifest.release_id ?? 'unknown'}`);
+  console.log(`git_head=${manifest.git_head}`);
+} catch (error) {
+  console.error(error.message);
+  process.exit(1);
+}