From 41a7747c09c9cb705f10c8ec27b4a7c198d9b1c1 Mon Sep 17 00:00:00 2001 From: john Date: Sat, 4 Jul 2026 13:41:17 +0800 Subject: [PATCH] fix(guard): make agent-run pause idempotent to stop portal restart loop Repeat guard checks no longer kickstart Portal when code runs are already disabled, preventing WeChat background replies from being killed every minute. Adds resume script and unit tests. Co-authored-by: Cursor --- scripts/agent-run-guard-lib.mjs | 77 ++++++++++++++++++++++++++++++++ scripts/agent-run-guard.mjs | 57 +++++++++++------------ scripts/agent-run-guard.test.mjs | 70 +++++++++++++++++++++++++++++ scripts/build-portal-runtime.mjs | 12 ++++- scripts/resume-agent-run-prod.sh | 58 ++++++++++++++++++++++++ 5 files changed, 245 insertions(+), 29 deletions(-) create mode 100644 scripts/agent-run-guard-lib.mjs create mode 100644 scripts/agent-run-guard.test.mjs create mode 100755 scripts/resume-agent-run-prod.sh diff --git a/scripts/agent-run-guard-lib.mjs b/scripts/agent-run-guard-lib.mjs new file mode 100644 index 0000000..416bca5 --- /dev/null +++ b/scripts/agent-run-guard-lib.mjs @@ -0,0 +1,77 @@ +export function readEnvValue(envRaw, key) { + for (const line of String(envRaw ?? '').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const idx = trimmed.indexOf('='); + if (idx < 0) continue; + const envKey = trimmed.slice(0, idx).trim(); + if (envKey !== key) continue; + let value = trimmed.slice(idx + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + return value; + } + return null; +} + +export function isGuardPaused(envRaw) { + const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'); + return value != null && String(value).trim() === '0'; +} + +export function evaluateHealth(queue) { + const positiveInt = (value, fallback) => { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; + }; + const thresholds = { + maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3), + maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000), + maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10), + maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000), + }; + const reasons = []; + if (queue.failedRecentCount >= thresholds.maxRecentFailures) { + reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`); + } + if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) { + reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`); + } + if (queue.queuedOrRetryable >= thresholds.maxPendingCount) { + reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`); + } + if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) { + reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`); + } + return { thresholds, reasons, shouldPause: reasons.length > 0 }; +} + +export function buildPausePlan({ shouldPause, envRaw, dryRun }) { + if (!shouldPause) { + return { + action: 'none', + alreadyPaused: isGuardPaused(envRaw), + applied: false, + actions: ['skip_pause: queue healthy'], + }; + } + if (isGuardPaused(envRaw)) { + return { + action: 'skip', + alreadyPaused: true, + applied: false, + actions: dryRun + ? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'] + : ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'], + }; + } + return { + action: 'apply', + alreadyPaused: false, + applied: false, + actions: dryRun + ? ['would apply first-time pause'] + : ['apply_pause: first trigger'], + }; +} diff --git a/scripts/agent-run-guard.mjs b/scripts/agent-run-guard.mjs index 8d1d507..0490083 100755 --- a/scripts/agent-run-guard.mjs +++ b/scripts/agent-run-guard.mjs @@ -4,6 +4,7 @@ import path from 'node:path'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import mysql from 'mysql2/promise'; +import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs'; const execFileAsync = promisify(execFile); const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker'; @@ -49,7 +50,11 @@ function printHelp() { ' node scripts/agent-run-guard.mjs [--dry-run]', ' node scripts/agent-run-guard.mjs --apply', '', - 'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.', + 'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env', + 'on the first pause trigger only. Repeat checks while already paused are no-ops.', + '', + 'Recovery after manual intervention:', + ' bash scripts/resume-agent-run-prod.sh', ].join('\n')); } @@ -228,29 +233,6 @@ async function readQueueHealth(now) { } } -function evaluateHealth(queue) { - const thresholds = { - maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3), - maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000), - maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10), - maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000), - }; - const reasons = []; - if (queue.failedRecentCount >= thresholds.maxRecentFailures) { - reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`); - } - if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) { - reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`); - } - if (queue.queuedOrRetryable >= thresholds.maxPendingCount) { - reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`); - } - if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) { - reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`); - } - return { thresholds, reasons, shouldPause: reasons.length > 0 }; -} - async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) { const actions = []; const updates = { @@ -304,6 +286,7 @@ if (args.help) { const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..'); const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env'); loadEnvFile(envFile); +const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : ''; const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null ? true @@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({ const evaluation = queue.error ? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false } : evaluateHealth(queue); -let pause = { applied: false, actions: [] }; +const pausePlan = buildPausePlan({ + shouldPause: evaluation.shouldPause, + envRaw, + dryRun: args.dryRun, +}); +let pause = { + applied: false, + skipped: pausePlan.action === 'skip', + alreadyPaused: pausePlan.alreadyPaused, + actions: [...pausePlan.actions], +}; -if (enabled && evaluation.shouldPause) { - pause = await disableCodeRuns({ +if (enabled && pausePlan.action === 'apply') { + const applied = await disableCodeRuns({ root, envFile, workerLabel, @@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) { reasons: evaluation.reasons, dryRun: args.dryRun, }); + pause = { + ...pause, + ...applied, + skipped: false, + alreadyPaused: false, + }; } const result = { - ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true, + ok: enabled + ? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun + : true, checkedAt: new Date(now).toISOString(), enabled, mode: args.apply ? 'apply' : 'dry-run', diff --git a/scripts/agent-run-guard.test.mjs b/scripts/agent-run-guard.test.mjs new file mode 100644 index 0000000..5ee47ae --- /dev/null +++ b/scripts/agent-run-guard.test.mjs @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + buildPausePlan, + evaluateHealth, + isGuardPaused, + readEnvValue, +} from './agent-run-guard-lib.mjs'; + +test('readEnvValue reads unquoted and quoted values', () => { + const raw = [ + '# comment', + 'MEMIND_AGENT_CODE_RUNS_ENABLED=1', + 'FOO="bar"', + ].join('\n'); + assert.equal(readEnvValue(raw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'), '1'); + assert.equal(readEnvValue(raw, 'FOO'), 'bar'); + assert.equal(readEnvValue(raw, 'MISSING'), null); +}); + +test('isGuardPaused is true only when code runs explicitly disabled', () => { + assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'), true); + assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=1\n'), false); + assert.equal(isGuardPaused('# no key\n'), false); +}); + +test('buildPausePlan skips repeat pause when env already disabled', () => { + const envRaw = 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'; + const plan = buildPausePlan({ + shouldPause: true, + envRaw, + dryRun: false, + }); + assert.equal(plan.action, 'skip'); + assert.equal(plan.alreadyPaused, true); + assert.match(plan.actions.join(' '), /skip_pause/); +}); + +test('buildPausePlan applies only on first pause trigger', () => { + const plan = buildPausePlan({ + shouldPause: true, + envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=1\n', + dryRun: false, + }); + assert.equal(plan.action, 'apply'); + assert.equal(plan.alreadyPaused, false); +}); + +test('buildPausePlan does nothing when queue is healthy', () => { + const plan = buildPausePlan({ + shouldPause: false, + envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n', + dryRun: false, + }); + assert.equal(plan.action, 'none'); +}); + +test('evaluateHealth flags stale pending and running heartbeats', () => { + const now = Date.now(); + const evaluation = evaluateHealth({ + failedRecentCount: 0, + oldestPendingAgeMs: 6 * 60 * 1000, + queuedOrRetryable: 1, + oldestRunningHeartbeatAgeMs: 16 * 60 * 1000, + }); + assert.equal(evaluation.shouldPause, true); + assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_pending_age_ms'))); + assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_running_heartbeat_age_ms'))); + assert.equal(now > 0, true); +}); diff --git a/scripts/build-portal-runtime.mjs b/scripts/build-portal-runtime.mjs index 64d417e..ec8b9c3 100755 --- a/scripts/build-portal-runtime.mjs +++ b/scripts/build-portal-runtime.mjs @@ -348,10 +348,18 @@ async function writeMetadata() { path.join(root, 'scripts', 'agent-run-guard.mjs'), path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), ); + await fs.copyFile( + path.join(root, 'scripts', 'agent-run-guard-lib.mjs'), + path.join(runtimeRoot, 'scripts', 'agent-run-guard-lib.mjs'), + ); await fs.copyFile( path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'), path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), ); + await fs.copyFile( + path.join(root, 'scripts', 'resume-agent-run-prod.sh'), + path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), + ); await writeFile( path.join(runtimeRoot, 'RUNBOOK.txt'), [ @@ -417,8 +425,9 @@ async function writeMetadata() { ' bash scripts/install-agent-run-worker-agent.sh # installs disabled by default', ' node scripts/check-agent-run-worker.mjs # read-only LaunchAgent/queue check', ' node scripts/agent-run-guard.mjs # dry-run auto-pause guard check', - ' node scripts/agent-run-guard.mjs --apply # stop worker and disable code-run gate when thresholds trip', + ' node scripts/agent-run-guard.mjs --apply # first-time pause only; repeat checks are no-ops', ' bash scripts/install-agent-run-guard-agent.sh # installs guard LaunchAgent', + ' bash scripts/resume-agent-run-prod.sh # manual recovery after guard pause', '', ].join('\n'), ); @@ -456,6 +465,7 @@ async function main() { await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755); + await fs.chmod(path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755); diff --git a/scripts/resume-agent-run-prod.sh b/scripts/resume-agent-run-prod.sh new file mode 100755 index 0000000..32b9756 --- /dev/null +++ b/scripts/resume-agent-run-prod.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +HOST="${MEMIND_PROD_HOST:-john@58.38.22.103}" +REMOTE_ROOT="${MEMIND_PROD_ROOT:-/Users/john/Project/Memind}" + +echo "[resume-agent-run] stop guard on ${HOST}" +ssh "${HOST}" "launchctl bootout gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true; launchctl disable gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true" + +echo "[resume-agent-run] enable code runs + clear stuck queue rows" +ssh "${HOST}" "bash -lc ' +set -euo pipefail +cd \"${REMOTE_ROOT}\" +set -a +source .env +set +a +if grep -q \"^MEMIND_AGENT_CODE_RUNS_ENABLED=\" .env; then + sed -i \"\" \"s/^MEMIND_AGENT_CODE_RUNS_ENABLED=.*/MEMIND_AGENT_CODE_RUNS_ENABLED=1/\" .env +else + echo \"MEMIND_AGENT_CODE_RUNS_ENABLED=1\" >> .env +fi +/opt/homebrew/opt/node@24/bin/node <<\"EOF\" +const mysql = require(\"mysql2/promise\"); +(async () => { + const pool = mysql.createPool(process.env.DATABASE_URL); + const now = Date.now(); + const [runs] = await pool.query( + \"UPDATE h5_agent_runs SET status=?, error_message=COALESCE(error_message, ?), updated_at=?, completed_at=? WHERE status IN (?, ?, ?)\", + [\"failed\", \"resume-agent-run: stale run cleared\", now, now, \"queued\", \"running\", \"retryable\"], + ); + const [msgs] = await pool.query( + \"UPDATE h5_wechat_mp_messages SET status=?, updated_at=? WHERE status=?\", + [\"failed\", now, \"processing\"], + ); + console.log(JSON.stringify({ agentRunsFailed: runs.affectedRows, wechatMessagesFailed: msgs.affectedRows })); + await pool.end(); +})().catch((err) => { + console.error(err); + process.exit(1); +}); +EOF +'" + +echo "[resume-agent-run] start worker + restart portal once" +ssh "${HOST}" "bash -lc ' +GUI=gui/\$(id -u) +launchctl enable \$GUI/cn.tkmind.memind-agent-run-worker 2>/dev/null || true +launchctl bootstrap \$GUI \$HOME/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist 2>/dev/null || true +launchctl kickstart -k \$GUI/cn.tkmind.memind-agent-run-worker || true +launchctl kickstart -k \$GUI/cn.tkmind.memind-portal +sleep 5 +curl -s -o /dev/null -w \"portal=%{http_code}\\n\" http://127.0.0.1:8081/api/status +'" + +echo "[resume-agent-run] done (guard remains stopped; re-enable manually after queue is healthy)" -- 2.52.0