scripts/agent-run-guard-lib.mjs

@@ -0,0 +1,77 @@
+export function readEnvValue(envRaw, key) {
+  for (const line of String(envRaw ?? '').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const idx = trimmed.indexOf('=');
+    if (idx < 0) continue;
+    const envKey = trimmed.slice(0, idx).trim();
+    if (envKey !== key) continue;
+    let value = trimmed.slice(idx + 1).trim();
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+      value = value.slice(1, -1);
+    }
+    return value;
+  }
+  return null;
+}
+
+export function isGuardPaused(envRaw) {
+  const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED');
+  return value != null && String(value).trim() === '0';
+}
+
+export function evaluateHealth(queue) {
+  const positiveInt = (value, fallback) => {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
+  };
+  const thresholds = {
+    maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
+    maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
+    maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
+    maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
+  };
+  const reasons = [];
+  if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
+    reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
+  }
+  if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
+    reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
+  }
+  if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
+    reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
+  }
+  if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
+    reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
+  }
+  return { thresholds, reasons, shouldPause: reasons.length > 0 };
+}
+
+export function buildPausePlan({ shouldPause, envRaw, dryRun }) {
+  if (!shouldPause) {
+    return {
+      action: 'none',
+      alreadyPaused: isGuardPaused(envRaw),
+      applied: false,
+      actions: ['skip_pause: queue healthy'],
+    };
+  }
+  if (isGuardPaused(envRaw)) {
+    return {
+      action: 'skip',
+      alreadyPaused: true,
+      applied: false,
+      actions: dryRun
+        ? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0']
+        : ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'],
+    };
+  }
+  return {
+    action: 'apply',
+    alreadyPaused: false,
+    applied: false,
+    actions: dryRun
+      ? ['would apply first-time pause']
+      : ['apply_pause: first trigger'],
+  };
+}

scripts/agent-run-guard.mjs

@@ -4,6 +4,7 @@ import path from 'node:path';
 import { execFile } from 'node:child_process';
 import { promisify } from 'node:util';
 import mysql from 'mysql2/promise';
+import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs';
 
 const execFileAsync = promisify(execFile);
 const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker';
@@ -49,7 +50,11 @@ function printHelp() {
     '  node scripts/agent-run-guard.mjs [--dry-run]',
     '  node scripts/agent-run-guard.mjs --apply',
     '',
-    'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.',
+    'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env',
+    'on the first pause trigger only. Repeat checks while already paused are no-ops.',
+    '',
+    'Recovery after manual intervention:',
+    '  bash scripts/resume-agent-run-prod.sh',
   ].join('\n'));
 }
 
@@ -228,29 +233,6 @@ async function readQueueHealth(now) {
   }
 }
 
-function evaluateHealth(queue) {
-  const thresholds = {
-    maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
-    maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
-    maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
-    maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
-  };
-  const reasons = [];
-  if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
-    reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
-  }
-  if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
-    reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
-  }
-  if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
-    reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
-  }
-  if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
-    reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
-  }
-  return { thresholds, reasons, shouldPause: reasons.length > 0 };
-}
-
 async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) {
   const actions = [];
   const updates = {
@@ -304,6 +286,7 @@ if (args.help) {
 const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
 const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env');
 loadEnvFile(envFile);
+const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : '';
 
 const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null
   ? true
@@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({
 const evaluation = queue.error
   ? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false }
   : evaluateHealth(queue);
-let pause = { applied: false, actions: [] };
+const pausePlan = buildPausePlan({
+  shouldPause: evaluation.shouldPause,
+  envRaw,
+  dryRun: args.dryRun,
+});
+let pause = {
+  applied: false,
+  skipped: pausePlan.action === 'skip',
+  alreadyPaused: pausePlan.alreadyPaused,
+  actions: [...pausePlan.actions],
+};
 
-if (enabled && evaluation.shouldPause) {
+if (enabled && pausePlan.action === 'apply') {
-  pause = await disableCodeRuns({
+  const applied = await disableCodeRuns({
     root,
     envFile,
     workerLabel,
@@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) {
     reasons: evaluation.reasons,
     dryRun: args.dryRun,
   });
+  pause = {
+    ...pause,
+    ...applied,
+    skipped: false,
+    alreadyPaused: false,
+  };
 }
 
 const result = {
-  ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true,
+  ok: enabled
+    ? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun
+    : true,
   checkedAt: new Date(now).toISOString(),
   enabled,
   mode: args.apply ? 'apply' : 'dry-run',

scripts/agent-run-guard.test.mjs

@@ -0,0 +1,70 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  buildPausePlan,
+  evaluateHealth,
+  isGuardPaused,
+  readEnvValue,
+} from './agent-run-guard-lib.mjs';
+
+test('readEnvValue reads unquoted and quoted values', () => {
+  const raw = [
+    '# comment',
+    'MEMIND_AGENT_CODE_RUNS_ENABLED=1',
+    'FOO="bar"',
+  ].join('\n');
+  assert.equal(readEnvValue(raw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'), '1');
+  assert.equal(readEnvValue(raw, 'FOO'), 'bar');
+  assert.equal(readEnvValue(raw, 'MISSING'), null);
+});
+
+test('isGuardPaused is true only when code runs explicitly disabled', () => {
+  assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'), true);
+  assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=1\n'), false);
+  assert.equal(isGuardPaused('# no key\n'), false);
+});
+
+test('buildPausePlan skips repeat pause when env already disabled', () => {
+  const envRaw = 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n';
+  const plan = buildPausePlan({
+    shouldPause: true,
+    envRaw,
+    dryRun: false,
+  });
+  assert.equal(plan.action, 'skip');
+  assert.equal(plan.alreadyPaused, true);
+  assert.match(plan.actions.join(' '), /skip_pause/);
+});
+
+test('buildPausePlan applies only on first pause trigger', () => {
+  const plan = buildPausePlan({
+    shouldPause: true,
+    envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=1\n',
+    dryRun: false,
+  });
+  assert.equal(plan.action, 'apply');
+  assert.equal(plan.alreadyPaused, false);
+});
+
+test('buildPausePlan does nothing when queue is healthy', () => {
+  const plan = buildPausePlan({
+    shouldPause: false,
+    envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n',
+    dryRun: false,
+  });
+  assert.equal(plan.action, 'none');
+});
+
+test('evaluateHealth flags stale pending and running heartbeats', () => {
+  const now = Date.now();
+  const evaluation = evaluateHealth({
+    failedRecentCount: 0,
+    oldestPendingAgeMs: 6 * 60 * 1000,
+    queuedOrRetryable: 1,
+    oldestRunningHeartbeatAgeMs: 16 * 60 * 1000,
+  });
+  assert.equal(evaluation.shouldPause, true);
+  assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_pending_age_ms')));
+  assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_running_heartbeat_age_ms')));
+  assert.equal(now > 0, true);
+});

scripts/build-portal-runtime.mjs

@@ -348,10 +348,18 @@ async function writeMetadata() {
     path.join(root, 'scripts', 'agent-run-guard.mjs'),
     path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'),
   );
+  await fs.copyFile(
+    path.join(root, 'scripts', 'agent-run-guard-lib.mjs'),
+    path.join(runtimeRoot, 'scripts', 'agent-run-guard-lib.mjs'),
+  );
   await fs.copyFile(
     path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'),
     path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'),
   );
+  await fs.copyFile(
+    path.join(root, 'scripts', 'resume-agent-run-prod.sh'),
+    path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'),
+  );
   await writeFile(
     path.join(runtimeRoot, 'RUNBOOK.txt'),
     [
@@ -417,8 +425,9 @@ async function writeMetadata() {
       '  bash scripts/install-agent-run-worker-agent.sh  # installs disabled by default',
       '  node scripts/check-agent-run-worker.mjs  # read-only LaunchAgent/queue check',
       '  node scripts/agent-run-guard.mjs  # dry-run auto-pause guard check',
-      '  node scripts/agent-run-guard.mjs --apply  # stop worker and disable code-run gate when thresholds trip',
+      '  node scripts/agent-run-guard.mjs --apply  # first-time pause only; repeat checks are no-ops',
       '  bash scripts/install-agent-run-guard-agent.sh  # installs guard LaunchAgent',
+      '  bash scripts/resume-agent-run-prod.sh  # manual recovery after guard pause',
       '',
     ].join('\n'),
   );
@@ -456,6 +465,7 @@ async function main() {
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755);
+  await fs.chmod(path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);

scripts/resume-agent-run-prod.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$ROOT"
+
+HOST="${MEMIND_PROD_HOST:-john@58.38.22.103}"
+REMOTE_ROOT="${MEMIND_PROD_ROOT:-/Users/john/Project/Memind}"
+
+echo "[resume-agent-run] stop guard on ${HOST}"
+ssh "${HOST}" "launchctl bootout gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true; launchctl disable gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true"
+
+echo "[resume-agent-run] enable code runs + clear stuck queue rows"
+ssh "${HOST}" "bash -lc '
+set -euo pipefail
+cd \"${REMOTE_ROOT}\"
+set -a
+source .env
+set +a
+if grep -q \"^MEMIND_AGENT_CODE_RUNS_ENABLED=\" .env; then
+  sed -i \"\" \"s/^MEMIND_AGENT_CODE_RUNS_ENABLED=.*/MEMIND_AGENT_CODE_RUNS_ENABLED=1/\" .env
+else
+  echo \"MEMIND_AGENT_CODE_RUNS_ENABLED=1\" >> .env
+fi
+/opt/homebrew/opt/node@24/bin/node <<\"EOF\"
+const mysql = require(\"mysql2/promise\");
+(async () => {
+  const pool = mysql.createPool(process.env.DATABASE_URL);
+  const now = Date.now();
+  const [runs] = await pool.query(
+    \"UPDATE h5_agent_runs SET status=?, error_message=COALESCE(error_message, ?), updated_at=?, completed_at=? WHERE status IN (?, ?, ?)\",
+    [\"failed\", \"resume-agent-run: stale run cleared\", now, now, \"queued\", \"running\", \"retryable\"],
+  );
+  const [msgs] = await pool.query(
+    \"UPDATE h5_wechat_mp_messages SET status=?, updated_at=? WHERE status=?\",
+    [\"failed\", now, \"processing\"],
+  );
+  console.log(JSON.stringify({ agentRunsFailed: runs.affectedRows, wechatMessagesFailed: msgs.affectedRows }));
+  await pool.end();
+})().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});
+EOF
+'"
+
+echo "[resume-agent-run] start worker + restart portal once"
+ssh "${HOST}" "bash -lc '
+GUI=gui/\$(id -u)
+launchctl enable \$GUI/cn.tkmind.memind-agent-run-worker 2>/dev/null || true
+launchctl bootstrap \$GUI \$HOME/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist 2>/dev/null || true
+launchctl kickstart -k \$GUI/cn.tkmind.memind-agent-run-worker || true
+launchctl kickstart -k \$GUI/cn.tkmind.memind-portal
+sleep 5
+curl -s -o /dev/null -w \"portal=%{http_code}\\n\" http://127.0.0.1:8081/api/status
+'"
+
+echo "[resume-agent-run] done (guard remains stopped; re-enable manually after queue is healthy)"