fix(release): harden stable promotion after canary acceptance
Memind CI / Test, build, and release guards (pull_request) Successful in 13m26s
Memind CI / Test, build, and release guards (pull_request) Successful in 13m26s
Verify matching 103 canary evidence before 8081 promotion, fix agent-run drain JSON parsing, tolerate macOS full-backup tar races, and extend goosed remount health waits for non-interactive SSH releases. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
@@ -84,10 +84,8 @@ if [[ "${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}" == "1" ]]; then
|
||||
fi
|
||||
|
||||
if [[ "${DRY_RUN}" -eq 0 ]]; then
|
||||
echo "当前脚本执行 8081 整包晋升，不是用户级灰度入口。" >&2
|
||||
echo "在同一候选完成 103 灰度验收且晋升证据校验落地前，禁止非 dry-run。" >&2
|
||||
echo "首次生产动作请使用 scripts/release-portal-canary-prod.sh。" >&2
|
||||
exit 1
|
||||
say "验证 103 灰度晋升证据"
|
||||
node "${ROOT}/scripts/verify-canary-promotion-evidence.mjs" --host "${HOST}"
|
||||
fi
|
||||
|
||||
say() {
|
||||
@@ -388,6 +386,7 @@ say "在 103 停旧服务并切换到无源码 runtime"
|
||||
ssh -o BatchMode=yes "${HOST}" \
|
||||
"RELEASE_ID='${RELEASE_ID}' APP_DIR='${APP_DIR}' INCOMING_DIR='${INCOMING_DIR}' BACKUP_DIR='${BACKUP_DIR}' ARCHIVE_DIR='${ARCHIVE_DIR}' HEALTH_URL='${HEALTH_URL}' PORTAL_LABEL='${PORTAL_LABEL}' PORTAL_TUNNEL_LABEL='${PORTAL_TUNNEL_LABEL}' MEMIND_PORTAL_TUNNEL_HOST='${MEMIND_PORTAL_TUNNEL_HOST}' MEMIND_PORTAL_TUNNEL_REMOTE_PORT='${MEMIND_PORTAL_TUNNEL_REMOTE_PORT}' LAUNCHD_GUI='${LAUNCHD_GUI}' ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES='${ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES:-0}' /bin/bash" <<'REMOTE_SCRIPT'
|
||||
set -euo pipefail
|
||||
export PATH="/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin:${PATH}"
|
||||
|
||||
RUNTIME_DIR="${APP_DIR}.runtime-${RELEASE_ID}"
|
||||
OLD_LIVE_DIR="${ARCHIVE_DIR}/Memind-source-before-${RELEASE_ID}"
|
||||
@@ -422,7 +421,7 @@ remount_goosed_after_live_swap() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
for _ in $(seq 1 60); do
|
||||
for _ in $(seq 1 120); do
|
||||
local healthy=1
|
||||
local containers=()
|
||||
while IFS= read -r container; do
|
||||
@@ -495,6 +494,11 @@ block_new_agent_runs_for_release() {
|
||||
touch "${APP_DIR}/.release-drain"
|
||||
}
|
||||
|
||||
read_agent_run_status_json() {
|
||||
local worker_script="$1"
|
||||
node "${worker_script}" --status 2>/dev/null | sed -n '/^{/,$p' || true
|
||||
}
|
||||
|
||||
drain_active_agent_runs() {
|
||||
local deadline=$(( $(date +%s) + ${MEMIND_RELEASE_DRAIN_TIMEOUT_SEC:-120} ))
|
||||
local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
|
||||
@@ -504,7 +508,7 @@ drain_active_agent_runs() {
|
||||
return 1
|
||||
fi
|
||||
local status_json
|
||||
status_json="$(node "${worker_script}" --status 2>/dev/null || true)"
|
||||
status_json="$(read_agent_run_status_json "${worker_script}")"
|
||||
if [[ -n "${status_json}" ]]; then
|
||||
local in_flight pending
|
||||
in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
|
||||
@@ -523,7 +527,7 @@ verify_no_active_agent_runs_before_swap() {
|
||||
local worker_script="${APP_DIR}/scripts/agent-run-worker.mjs"
|
||||
[[ -f "${worker_script}" ]] || return 1
|
||||
local status_json in_flight pending
|
||||
status_json="$(node "${worker_script}" --status 2>/dev/null || true)"
|
||||
status_json="$(read_agent_run_status_json "${worker_script}")"
|
||||
in_flight="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.inFlight||0)); } catch { console.log("unknown"); } });')"
|
||||
pending="$(printf '%s' "${status_json}" | node --input-type=module -e 'let s=""; process.stdin.on("data", c => s += c); process.stdin.on("end", () => { try { const q=JSON.parse(s).queue||{}; console.log(Number(q.pendingDispatches||0)); } catch { console.log("unknown"); } });')"
|
||||
[[ "${in_flight}" == "0" && "${pending}" == "0" ]]
|
||||
@@ -652,8 +656,18 @@ copy_persisted_item() {
|
||||
}
|
||||
|
||||
say "备份当前 live 全目录"
|
||||
set +e
|
||||
COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
|
||||
-czf "${FULL_BACKUP_TAR}" -C "$(dirname "${APP_DIR}")" "$(basename "${APP_DIR}")"
|
||||
tar_status=$?
|
||||
set -e
|
||||
if [[ ! -s "${FULL_BACKUP_TAR}" ]]; then
|
||||
echo "full backup failed: ${FULL_BACKUP_TAR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${tar_status}" -ne 0 ]]; then
|
||||
echo "warning: full backup tar reported errors; continuing with partial archive" >&2
|
||||
fi
|
||||
|
||||
say "单独备份持久目录"
|
||||
tmp_persist_dir="$(mktemp -d "${TMPDIR:-/tmp}/memind-persisted.XXXXXX")"
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(new URL('..', import.meta.url).pathname);
|
||||
const REMOTE_ROOT = process.env.STUDIO_REMOTE_ROOT ?? '/Users/john/Project';
|
||||
const CANDIDATE_BASE = `${REMOTE_ROOT}/Memind-candidates`;
|
||||
|
||||
const REMOTE_SCRIPT = [
|
||||
'set -euo pipefail',
|
||||
'expected_commit="$1"',
|
||||
'candidate_base="$2"',
|
||||
'matched=""',
|
||||
'latest=""',
|
||||
'for dir in "${candidate_base}"/*; do',
|
||||
' [[ -d "${dir}" ]] || continue',
|
||||
' [[ -f "${dir}/.release-manifest.txt" ]] || continue',
|
||||
' if [[ -z "${latest}" || "$(basename "${dir}")" > "$(basename "${latest}")" ]]; then',
|
||||
' latest="${dir}"',
|
||||
' fi',
|
||||
' if grep -q "^git_head=${expected_commit}$" "${dir}/.release-manifest.txt"; then',
|
||||
' matched="${dir}"',
|
||||
' fi',
|
||||
'done',
|
||||
'if [[ -n "${matched}" ]]; then candidate="${matched}"; else candidate="${latest}"; fi',
|
||||
'[[ -n "${candidate}" ]] || { echo "NO_CANDIDATE"; exit 1; }',
|
||||
'printf \'candidate_dir=%s\\n\' "${candidate}"',
|
||||
'cat "${candidate}/.release-manifest.txt"',
|
||||
'printf \'\\n---HEALTH---\\n\'',
|
||||
'curl -s -D - -o /dev/null http://127.0.0.1:18081/api/status | tr -d \'\\r\'',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
host: process.env.STUDIO_HOST ?? '58.38.22.103',
|
||||
commit: null,
|
||||
};
|
||||
for (let index = 2; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--host') options.host = argv[++index] ?? '';
|
||||
else if (arg === '--commit') options.commit = argv[++index] ?? '';
|
||||
else throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function git(...args) {
|
||||
const result = spawnSync('git', args, { cwd: ROOT, encoding: 'utf8' });
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function ssh(host, args, input = '') {
|
||||
const result = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=15', host, ...args], {
|
||||
encoding: 'utf8',
|
||||
input,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(`SSH to ${host} failed:\n${result.stderr || result.stdout}`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function parseManifest(content) {
|
||||
const fields = {};
|
||||
for (const line of content.split('\n')) {
|
||||
const separator = line.indexOf('=');
|
||||
if (separator <= 0) continue;
|
||||
fields[line.slice(0, separator)] = line.slice(separator + 1);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
try {
|
||||
const options = parseArgs(process.argv);
|
||||
const commit = options.commit ?? git('rev-parse', 'HEAD');
|
||||
const remoteUserHost = options.host.includes('@') ? options.host : `john@${options.host}`;
|
||||
|
||||
const remoteOutput = ssh(
|
||||
remoteUserHost,
|
||||
['/bin/bash', '-s', '--', commit, CANDIDATE_BASE],
|
||||
REMOTE_SCRIPT,
|
||||
);
|
||||
|
||||
const healthMarker = '\n---HEALTH---\n';
|
||||
const markerIndex = remoteOutput.indexOf('---HEALTH---');
|
||||
if (markerIndex < 0) {
|
||||
throw new Error('Canary promotion evidence response missing health probe');
|
||||
}
|
||||
const manifestText = remoteOutput.slice(0, markerIndex);
|
||||
const healthText = remoteOutput.slice(markerIndex + healthMarker.trim().length);
|
||||
|
||||
const candidateDir = manifestText
|
||||
.split('\n')
|
||||
.find((line) => line.startsWith('candidate_dir='))
|
||||
?.slice('candidate_dir='.length);
|
||||
const manifest = parseManifest(
|
||||
manifestText
|
||||
.split('\n')
|
||||
.filter((line) => !line.startsWith('candidate_dir='))
|
||||
.join('\n'),
|
||||
);
|
||||
|
||||
if (!candidateDir) {
|
||||
throw new Error('103 has no canary candidate directory');
|
||||
}
|
||||
if (manifest.git_head !== commit) {
|
||||
throw new Error(
|
||||
`Canary candidate commit mismatch: candidate=${manifest.git_head ?? 'missing'} expected=${commit}`,
|
||||
);
|
||||
}
|
||||
if (!/^HTTP\/1\.1 200/m.test(healthText)) {
|
||||
throw new Error('Canary candidate /api/status is not healthy on 18081');
|
||||
}
|
||||
if (!/^x-memind-runtime-role: candidate/im.test(healthText)) {
|
||||
throw new Error('Canary candidate health probe missing X-Memind-Runtime-Role: candidate');
|
||||
}
|
||||
|
||||
console.log('Canary promotion evidence verified');
|
||||
console.log(`candidate_dir=${candidateDir}`);
|
||||
console.log(`release_id=${manifest.release_id ?? 'unknown'}`);
|
||||
console.log(`git_head=${manifest.git_head}`);
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user