tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(guard): make agent-run pause idempotent to stop portal restart loop

Browse Source 
Repeat guard checks no longer kickstart Portal when code runs are already disabled, preventing WeChat background replies from being killed every minute. Adds resume script and unit tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-07-04 13:41:17 +08:00
parent 2a3579c73a
commit 41a7747c09
5 changed files with 245 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agent-run-guard-lib.mjs
+77
View File
@@ -0,0 +1,77 @@
export function readEnvValue(envRaw, key) {
for (const line of String(envRaw ?? '').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const idx = trimmed.indexOf('=');
if (idx < 0) continue;
const envKey = trimmed.slice(0, idx).trim();
if (envKey !== key) continue;
let value = trimmed.slice(idx + 1).trim();
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
return value;
}
return null;
}
export function isGuardPaused(envRaw) {
const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED');
return value != null && String(value).trim() === '0';
}
export function evaluateHealth(queue) {
const positiveInt = (value, fallback) => {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
};
const thresholds = {
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
};
const reasons = [];
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
}
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
}
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
export function buildPausePlan({ shouldPause, envRaw, dryRun }) {
if (!shouldPause) {
return {
action: 'none',
alreadyPaused: isGuardPaused(envRaw),
applied: false,
actions: ['skip_pause: queue healthy'],
};
}
if (isGuardPaused(envRaw)) {
return {
action: 'skip',
alreadyPaused: true,
applied: false,
actions: dryRun
? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0']
: ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'],
};
}
return {
action: 'apply',
alreadyPaused: false,
applied: false,
actions: dryRun
? ['would apply first-time pause']
: ['apply_pause: first trigger'],
};
}
scripts/agent-run-guard.mjs
+29 -28
View File
@@ -4,6 +4,7 @@ import path from 'node:path';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import mysql from 'mysql2/promise';
import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs';
const execFileAsync = promisify(execFile);
const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker';
@@ -49,7 +50,11 @@ function printHelp() {
' node scripts/agent-run-guard.mjs [--dry-run]',
' node scripts/agent-run-guard.mjs --apply',
'',
'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.',
'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env',
'on the first pause trigger only. Repeat checks while already paused are no-ops.',
'',
'Recovery after manual intervention:',
' bash scripts/resume-agent-run-prod.sh',
].join('\n'));
}
@@ -228,29 +233,6 @@ async function readQueueHealth(now) {
}
}
function evaluateHealth(queue) {
const thresholds = {
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
};
const reasons = [];
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
}
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
}
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) {
const actions = [];
const updates = {
@@ -304,6 +286,7 @@ if (args.help) {
const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env');
loadEnvFile(envFile);
const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : '';
const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null
? true
@@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({
const evaluation = queue.error
? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false }
: evaluateHealth(queue);
let pause = { applied: false, actions: [] };
const pausePlan = buildPausePlan({
shouldPause: evaluation.shouldPause,
envRaw,
dryRun: args.dryRun,
});
let pause = {
applied: false,
skipped: pausePlan.action === 'skip',
alreadyPaused: pausePlan.alreadyPaused,
actions: [...pausePlan.actions],
};
if (enabled && evaluation.shouldPause) {
pause = await disableCodeRuns({
if (enabled && pausePlan.action === 'apply') {
const applied = await disableCodeRuns({
root,
envFile,
workerLabel,
@@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) {
reasons: evaluation.reasons,
dryRun: args.dryRun,
});
pause = {
...pause,
...applied,
skipped: false,
alreadyPaused: false,
};
}
const result = {
ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true,
ok: enabled
? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun
: true,
checkedAt: new Date(now).toISOString(),
enabled,
mode: args.apply ? 'apply' : 'dry-run',
scripts/agent-run-guard.test.mjs
+70
View File
@@ -0,0 +1,70 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
buildPausePlan,
evaluateHealth,
isGuardPaused,
readEnvValue,
} from './agent-run-guard-lib.mjs';
test('readEnvValue reads unquoted and quoted values', () => {
const raw = [
'# comment',
'MEMIND_AGENT_CODE_RUNS_ENABLED=1',
'FOO="bar"',
].join('\n');
assert.equal(readEnvValue(raw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'), '1');
assert.equal(readEnvValue(raw, 'FOO'), 'bar');
assert.equal(readEnvValue(raw, 'MISSING'), null);
});
test('isGuardPaused is true only when code runs explicitly disabled', () => {
assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'), true);
assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=1\n'), false);
assert.equal(isGuardPaused('# no key\n'), false);
});
test('buildPausePlan skips repeat pause when env already disabled', () => {
const envRaw = 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n';
const plan = buildPausePlan({
shouldPause: true,
envRaw,
dryRun: false,
});
assert.equal(plan.action, 'skip');
assert.equal(plan.alreadyPaused, true);
assert.match(plan.actions.join(' '), /skip_pause/);
});
test('buildPausePlan applies only on first pause trigger', () => {
const plan = buildPausePlan({
shouldPause: true,
envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=1\n',
dryRun: false,
});
assert.equal(plan.action, 'apply');
assert.equal(plan.alreadyPaused, false);
});
test('buildPausePlan does nothing when queue is healthy', () => {
const plan = buildPausePlan({
shouldPause: false,
envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n',
dryRun: false,
});
assert.equal(plan.action, 'none');
});
test('evaluateHealth flags stale pending and running heartbeats', () => {
const now = Date.now();
const evaluation = evaluateHealth({
failedRecentCount: 0,
oldestPendingAgeMs: 6 * 60 * 1000,
queuedOrRetryable: 1,
oldestRunningHeartbeatAgeMs: 16 * 60 * 1000,
});
assert.equal(evaluation.shouldPause, true);
assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_pending_age_ms')));
assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_running_heartbeat_age_ms')));
assert.equal(now > 0, true);
});
scripts/build-portal-runtime.mjs
+11 -1
View File
@@ -348,10 +348,18 @@ async function writeMetadata() {
path.join(root, 'scripts', 'agent-run-guard.mjs'),
path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'),
);
await fs.copyFile(
path.join(root, 'scripts', 'agent-run-guard-lib.mjs'),
path.join(runtimeRoot, 'scripts', 'agent-run-guard-lib.mjs'),
);
await fs.copyFile(
path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'),
path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'),
);
await fs.copyFile(
path.join(root, 'scripts', 'resume-agent-run-prod.sh'),
path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'),
);
await writeFile(
path.join(runtimeRoot, 'RUNBOOK.txt'),
[
@@ -417,8 +425,9 @@ async function writeMetadata() {
' bash scripts/install-agent-run-worker-agent.sh # installs disabled by default',
' node scripts/check-agent-run-worker.mjs # read-only LaunchAgent/queue check',
' node scripts/agent-run-guard.mjs # dry-run auto-pause guard check',
' node scripts/agent-run-guard.mjs --apply # stop worker and disable code-run gate when thresholds trip',
' node scripts/agent-run-guard.mjs --apply # first-time pause only; repeat checks are no-ops',
' bash scripts/install-agent-run-guard-agent.sh # installs guard LaunchAgent',
' bash scripts/resume-agent-run-prod.sh # manual recovery after guard pause',
'',
].join('\n'),
);
@@ -456,6 +465,7 @@ async function main() {
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
scripts/resume-agent-run-prod.sh
Executable
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
HOST="${MEMIND_PROD_HOST:-john@58.38.22.103}"
REMOTE_ROOT="${MEMIND_PROD_ROOT:-/Users/john/Project/Memind}"
echo "[resume-agent-run] stop guard on ${HOST}"
ssh "${HOST}" "launchctl bootout gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true; launchctl disable gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true"
echo "[resume-agent-run] enable code runs + clear stuck queue rows"
ssh "${HOST}" "bash -lc '
set -euo pipefail
cd \"${REMOTE_ROOT}\"
set -a
source .env
set +a
if grep -q \"^MEMIND_AGENT_CODE_RUNS_ENABLED=\" .env; then
sed -i \"\" \"s/^MEMIND_AGENT_CODE_RUNS_ENABLED=.*/MEMIND_AGENT_CODE_RUNS_ENABLED=1/\" .env
else
echo \"MEMIND_AGENT_CODE_RUNS_ENABLED=1\" >> .env
fi
/opt/homebrew/opt/node@24/bin/node <<\"EOF\"
const mysql = require(\"mysql2/promise\");
(async () => {
const pool = mysql.createPool(process.env.DATABASE_URL);
const now = Date.now();
const [runs] = await pool.query(
\"UPDATE h5_agent_runs SET status=?, error_message=COALESCE(error_message, ?), updated_at=?, completed_at=? WHERE status IN (?, ?, ?)\",
[\"failed\", \"resume-agent-run: stale run cleared\", now, now, \"queued\", \"running\", \"retryable\"],
);
const [msgs] = await pool.query(
\"UPDATE h5_wechat_mp_messages SET status=?, updated_at=? WHERE status=?\",
[\"failed\", now, \"processing\"],
);
console.log(JSON.stringify({ agentRunsFailed: runs.affectedRows, wechatMessagesFailed: msgs.affectedRows }));
await pool.end();
})().catch((err) => {
console.error(err);
process.exit(1);
});
EOF
'"
echo "[resume-agent-run] start worker + restart portal once"
ssh "${HOST}" "bash -lc '
GUI=gui/\$(id -u)
launchctl enable \$GUI/cn.tkmind.memind-agent-run-worker 2>/dev/null || true
launchctl bootstrap \$GUI \$HOME/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist 2>/dev/null || true
launchctl kickstart -k \$GUI/cn.tkmind.memind-agent-run-worker || true
launchctl kickstart -k \$GUI/cn.tkmind.memind-portal
sleep 5
curl -s -o /dev/null -w \"portal=%{http_code}\\n\" http://127.0.0.1:8081/api/status
'"
echo "[resume-agent-run] done (guard remains stopped; re-enable manually after queue is healthy)"