diff --git a/scripts/agent-run-guard-lib.mjs b/scripts/agent-run-guard-lib.mjs new file mode 100644 index 0000000..416bca5 --- /dev/null +++ b/scripts/agent-run-guard-lib.mjs @@ -0,0 +1,77 @@ +export function readEnvValue(envRaw, key) { + for (const line of String(envRaw ?? '').split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const idx = trimmed.indexOf('='); + if (idx < 0) continue; + const envKey = trimmed.slice(0, idx).trim(); + if (envKey !== key) continue; + let value = trimmed.slice(idx + 1).trim(); + if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) { + value = value.slice(1, -1); + } + return value; + } + return null; +} + +export function isGuardPaused(envRaw) { + const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'); + return value != null && String(value).trim() === '0'; +} + +export function evaluateHealth(queue) { + const positiveInt = (value, fallback) => { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; + }; + const thresholds = { + maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3), + maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000), + maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10), + maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000), + }; + const reasons = []; + if (queue.failedRecentCount >= thresholds.maxRecentFailures) { + reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`); + } + if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) { + reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`); + } + if (queue.queuedOrRetryable >= thresholds.maxPendingCount) { + reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`); + } + if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) { + reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`); + } + return { thresholds, reasons, shouldPause: reasons.length > 0 }; +} + +export function buildPausePlan({ shouldPause, envRaw, dryRun }) { + if (!shouldPause) { + return { + action: 'none', + alreadyPaused: isGuardPaused(envRaw), + applied: false, + actions: ['skip_pause: queue healthy'], + }; + } + if (isGuardPaused(envRaw)) { + return { + action: 'skip', + alreadyPaused: true, + applied: false, + actions: dryRun + ? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'] + : ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'], + }; + } + return { + action: 'apply', + alreadyPaused: false, + applied: false, + actions: dryRun + ? ['would apply first-time pause'] + : ['apply_pause: first trigger'], + }; +} diff --git a/scripts/agent-run-guard.mjs b/scripts/agent-run-guard.mjs index 8d1d507..0490083 100755 --- a/scripts/agent-run-guard.mjs +++ b/scripts/agent-run-guard.mjs @@ -4,6 +4,7 @@ import path from 'node:path'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; import mysql from 'mysql2/promise'; +import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs'; const execFileAsync = promisify(execFile); const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker'; @@ -49,7 +50,11 @@ function printHelp() { ' node scripts/agent-run-guard.mjs [--dry-run]', ' node scripts/agent-run-guard.mjs --apply', '', - 'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.', + 'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env', + 'on the first pause trigger only. Repeat checks while already paused are no-ops.', + '', + 'Recovery after manual intervention:', + ' bash scripts/resume-agent-run-prod.sh', ].join('\n')); } @@ -228,29 +233,6 @@ async function readQueueHealth(now) { } } -function evaluateHealth(queue) { - const thresholds = { - maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3), - maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000), - maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10), - maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000), - }; - const reasons = []; - if (queue.failedRecentCount >= thresholds.maxRecentFailures) { - reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`); - } - if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) { - reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`); - } - if (queue.queuedOrRetryable >= thresholds.maxPendingCount) { - reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`); - } - if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) { - reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`); - } - return { thresholds, reasons, shouldPause: reasons.length > 0 }; -} - async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) { const actions = []; const updates = { @@ -304,6 +286,7 @@ if (args.help) { const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..'); const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env'); loadEnvFile(envFile); +const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : ''; const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null ? true @@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({ const evaluation = queue.error ? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false } : evaluateHealth(queue); -let pause = { applied: false, actions: [] }; +const pausePlan = buildPausePlan({ + shouldPause: evaluation.shouldPause, + envRaw, + dryRun: args.dryRun, +}); +let pause = { + applied: false, + skipped: pausePlan.action === 'skip', + alreadyPaused: pausePlan.alreadyPaused, + actions: [...pausePlan.actions], +}; -if (enabled && evaluation.shouldPause) { - pause = await disableCodeRuns({ +if (enabled && pausePlan.action === 'apply') { + const applied = await disableCodeRuns({ root, envFile, workerLabel, @@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) { reasons: evaluation.reasons, dryRun: args.dryRun, }); + pause = { + ...pause, + ...applied, + skipped: false, + alreadyPaused: false, + }; } const result = { - ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true, + ok: enabled + ? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun + : true, checkedAt: new Date(now).toISOString(), enabled, mode: args.apply ? 'apply' : 'dry-run', diff --git a/scripts/agent-run-guard.test.mjs b/scripts/agent-run-guard.test.mjs new file mode 100644 index 0000000..5ee47ae --- /dev/null +++ b/scripts/agent-run-guard.test.mjs @@ -0,0 +1,70 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + buildPausePlan, + evaluateHealth, + isGuardPaused, + readEnvValue, +} from './agent-run-guard-lib.mjs'; + +test('readEnvValue reads unquoted and quoted values', () => { + const raw = [ + '# comment', + 'MEMIND_AGENT_CODE_RUNS_ENABLED=1', + 'FOO="bar"', + ].join('\n'); + assert.equal(readEnvValue(raw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'), '1'); + assert.equal(readEnvValue(raw, 'FOO'), 'bar'); + assert.equal(readEnvValue(raw, 'MISSING'), null); +}); + +test('isGuardPaused is true only when code runs explicitly disabled', () => { + assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'), true); + assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=1\n'), false); + assert.equal(isGuardPaused('# no key\n'), false); +}); + +test('buildPausePlan skips repeat pause when env already disabled', () => { + const envRaw = 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'; + const plan = buildPausePlan({ + shouldPause: true, + envRaw, + dryRun: false, + }); + assert.equal(plan.action, 'skip'); + assert.equal(plan.alreadyPaused, true); + assert.match(plan.actions.join(' '), /skip_pause/); +}); + +test('buildPausePlan applies only on first pause trigger', () => { + const plan = buildPausePlan({ + shouldPause: true, + envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=1\n', + dryRun: false, + }); + assert.equal(plan.action, 'apply'); + assert.equal(plan.alreadyPaused, false); +}); + +test('buildPausePlan does nothing when queue is healthy', () => { + const plan = buildPausePlan({ + shouldPause: false, + envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n', + dryRun: false, + }); + assert.equal(plan.action, 'none'); +}); + +test('evaluateHealth flags stale pending and running heartbeats', () => { + const now = Date.now(); + const evaluation = evaluateHealth({ + failedRecentCount: 0, + oldestPendingAgeMs: 6 * 60 * 1000, + queuedOrRetryable: 1, + oldestRunningHeartbeatAgeMs: 16 * 60 * 1000, + }); + assert.equal(evaluation.shouldPause, true); + assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_pending_age_ms'))); + assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_running_heartbeat_age_ms'))); + assert.equal(now > 0, true); +}); diff --git a/scripts/build-portal-runtime.mjs b/scripts/build-portal-runtime.mjs index 64d417e..ec8b9c3 100755 --- a/scripts/build-portal-runtime.mjs +++ b/scripts/build-portal-runtime.mjs @@ -348,10 +348,18 @@ async function writeMetadata() { path.join(root, 'scripts', 'agent-run-guard.mjs'), path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), ); + await fs.copyFile( + path.join(root, 'scripts', 'agent-run-guard-lib.mjs'), + path.join(runtimeRoot, 'scripts', 'agent-run-guard-lib.mjs'), + ); await fs.copyFile( path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'), path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), ); + await fs.copyFile( + path.join(root, 'scripts', 'resume-agent-run-prod.sh'), + path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), + ); await writeFile( path.join(runtimeRoot, 'RUNBOOK.txt'), [ @@ -417,8 +425,9 @@ async function writeMetadata() { ' bash scripts/install-agent-run-worker-agent.sh # installs disabled by default', ' node scripts/check-agent-run-worker.mjs # read-only LaunchAgent/queue check', ' node scripts/agent-run-guard.mjs # dry-run auto-pause guard check', - ' node scripts/agent-run-guard.mjs --apply # stop worker and disable code-run gate when thresholds trip', + ' node scripts/agent-run-guard.mjs --apply # first-time pause only; repeat checks are no-ops', ' bash scripts/install-agent-run-guard-agent.sh # installs guard LaunchAgent', + ' bash scripts/resume-agent-run-prod.sh # manual recovery after guard pause', '', ].join('\n'), ); @@ -456,6 +465,7 @@ async function main() { await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755); + await fs.chmod(path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755); await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755); diff --git a/scripts/resume-agent-run-prod.sh b/scripts/resume-agent-run-prod.sh new file mode 100755 index 0000000..32b9756 --- /dev/null +++ b/scripts/resume-agent-run-prod.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +HOST="${MEMIND_PROD_HOST:-john@58.38.22.103}" +REMOTE_ROOT="${MEMIND_PROD_ROOT:-/Users/john/Project/Memind}" + +echo "[resume-agent-run] stop guard on ${HOST}" +ssh "${HOST}" "launchctl bootout gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true; launchctl disable gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true" + +echo "[resume-agent-run] enable code runs + clear stuck queue rows" +ssh "${HOST}" "bash -lc ' +set -euo pipefail +cd \"${REMOTE_ROOT}\" +set -a +source .env +set +a +if grep -q \"^MEMIND_AGENT_CODE_RUNS_ENABLED=\" .env; then + sed -i \"\" \"s/^MEMIND_AGENT_CODE_RUNS_ENABLED=.*/MEMIND_AGENT_CODE_RUNS_ENABLED=1/\" .env +else + echo \"MEMIND_AGENT_CODE_RUNS_ENABLED=1\" >> .env +fi +/opt/homebrew/opt/node@24/bin/node <<\"EOF\" +const mysql = require(\"mysql2/promise\"); +(async () => { + const pool = mysql.createPool(process.env.DATABASE_URL); + const now = Date.now(); + const [runs] = await pool.query( + \"UPDATE h5_agent_runs SET status=?, error_message=COALESCE(error_message, ?), updated_at=?, completed_at=? WHERE status IN (?, ?, ?)\", + [\"failed\", \"resume-agent-run: stale run cleared\", now, now, \"queued\", \"running\", \"retryable\"], + ); + const [msgs] = await pool.query( + \"UPDATE h5_wechat_mp_messages SET status=?, updated_at=? WHERE status=?\", + [\"failed\", now, \"processing\"], + ); + console.log(JSON.stringify({ agentRunsFailed: runs.affectedRows, wechatMessagesFailed: msgs.affectedRows })); + await pool.end(); +})().catch((err) => { + console.error(err); + process.exit(1); +}); +EOF +'" + +echo "[resume-agent-run] start worker + restart portal once" +ssh "${HOST}" "bash -lc ' +GUI=gui/\$(id -u) +launchctl enable \$GUI/cn.tkmind.memind-agent-run-worker 2>/dev/null || true +launchctl bootstrap \$GUI \$HOME/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist 2>/dev/null || true +launchctl kickstart -k \$GUI/cn.tkmind.memind-agent-run-worker || true +launchctl kickstart -k \$GUI/cn.tkmind.memind-portal +sleep 5 +curl -s -o /dev/null -w \"portal=%{http_code}\\n\" http://127.0.0.1:8081/api/status +'" + +echo "[resume-agent-run] done (guard remains stopped; re-enable manually after queue is healthy)"