tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(guard): idempotent agent-run pause to stop portal restart loop

Browse Source 
Skip repeat guard pause actions when code runs already disabled; add resume script and tests.
This commit was merged in pull request #3.
This commit is contained in:
tkmind
2026-07-04 05:41:51 +00:00
parent 2a3579c73a
commit fafc1fe7fd
5 changed files with 245 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agent-run-guard-lib.mjs
+77
View File
@@ -0,0 +1,77 @@
export function readEnvValue(envRaw, key) {
for (const line of String(envRaw ?? '').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const idx = trimmed.indexOf('=');
if (idx < 0) continue;
const envKey = trimmed.slice(0, idx).trim();
if (envKey !== key) continue;
let value = trimmed.slice(idx + 1).trim();
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
value = value.slice(1, -1);
}
return value;
}
return null;
}
export function isGuardPaused(envRaw) {
const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED');
return value != null && String(value).trim() === '0';
}
export function evaluateHealth(queue) {
const positiveInt = (value, fallback) => {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
};
const thresholds = {
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
};
const reasons = [];
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
}
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
}
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
export function buildPausePlan({ shouldPause, envRaw, dryRun }) {
if (!shouldPause) {
return {
action: 'none',
alreadyPaused: isGuardPaused(envRaw),
applied: false,
actions: ['skip_pause: queue healthy'],
};
}
if (isGuardPaused(envRaw)) {
return {
action: 'skip',
alreadyPaused: true,
applied: false,
actions: dryRun
? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0']
: ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'],
};
}
return {
action: 'apply',
alreadyPaused: false,
applied: false,
actions: dryRun
? ['would apply first-time pause']
: ['apply_pause: first trigger'],
};
}
scripts/agent-run-guard.mjs
+29 -28
View File
@@ -4,6 +4,7 @@ import path from 'node:path';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import mysql from 'mysql2/promise';
import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs';
const execFileAsync = promisify(execFile);
const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker';
@@ -49,7 +50,11 @@ function printHelp() {
' node scripts/agent-run-guard.mjs [--dry-run]',
' node scripts/agent-run-guard.mjs --apply',
'',
'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.',
'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env',
'on the first pause trigger only. Repeat checks while already paused are no-ops.',
'',
'Recovery after manual intervention:',
' bash scripts/resume-agent-run-prod.sh',
].join('\n'));
}
@@ -228,29 +233,6 @@ async function readQueueHealth(now) {
}
}
function evaluateHealth(queue) {
const thresholds = {
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
};
const reasons = [];
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
}
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
}
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) {
const actions = [];
const updates = {
@@ -304,6 +286,7 @@ if (args.help) {
const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env');
loadEnvFile(envFile);
const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : '';
const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null
? true
@@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({
const evaluation = queue.error
? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false }
: evaluateHealth(queue);
let pause = { applied: false, actions: [] };
const pausePlan = buildPausePlan({
shouldPause: evaluation.shouldPause,
envRaw,
dryRun: args.dryRun,
});
let pause = {
applied: false,
skipped: pausePlan.action === 'skip',
alreadyPaused: pausePlan.alreadyPaused,
actions: [...pausePlan.actions],
};
if (enabled && evaluation.shouldPause) {
pause = await disableCodeRuns({
if (enabled && pausePlan.action === 'apply') {
const applied = await disableCodeRuns({
root,
envFile,
workerLabel,
@@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) {
reasons: evaluation.reasons,
dryRun: args.dryRun,
});
pause = {
...pause,
...applied,
skipped: false,
alreadyPaused: false,
};
}
const result = {
ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true,
ok: enabled
? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun
: true,
checkedAt: new Date(now).toISOString(),
enabled,
mode: args.apply ? 'apply' : 'dry-run',
scripts/agent-run-guard.test.mjs
+70
View File
@@ -0,0 +1,70 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import {
buildPausePlan,
evaluateHealth,
isGuardPaused,
readEnvValue,
} from './agent-run-guard-lib.mjs';
test('readEnvValue reads unquoted and quoted values', () => {
const raw = [
'# comment',
'MEMIND_AGENT_CODE_RUNS_ENABLED=1',
'FOO="bar"',
].join('\n');
assert.equal(readEnvValue(raw, 'MEMIND_AGENT_CODE_RUNS_ENABLED'), '1');
assert.equal(readEnvValue(raw, 'FOO'), 'bar');
assert.equal(readEnvValue(raw, 'MISSING'), null);
});
test('isGuardPaused is true only when code runs explicitly disabled', () => {
assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=0\n'), true);
assert.equal(isGuardPaused('MEMIND_AGENT_CODE_RUNS_ENABLED=1\n'), false);
assert.equal(isGuardPaused('# no key\n'), false);
});
test('buildPausePlan skips repeat pause when env already disabled', () => {
const envRaw = 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n';
const plan = buildPausePlan({
shouldPause: true,
envRaw,
dryRun: false,
});
assert.equal(plan.action, 'skip');
assert.equal(plan.alreadyPaused, true);
assert.match(plan.actions.join(' '), /skip_pause/);
});
test('buildPausePlan applies only on first pause trigger', () => {
const plan = buildPausePlan({
shouldPause: true,
envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=1\n',
dryRun: false,
});
assert.equal(plan.action, 'apply');
assert.equal(plan.alreadyPaused, false);
});
test('buildPausePlan does nothing when queue is healthy', () => {
const plan = buildPausePlan({
shouldPause: false,
envRaw: 'MEMIND_AGENT_CODE_RUNS_ENABLED=0\n',
dryRun: false,
});
assert.equal(plan.action, 'none');
});
test('evaluateHealth flags stale pending and running heartbeats', () => {
const now = Date.now();
const evaluation = evaluateHealth({
failedRecentCount: 0,
oldestPendingAgeMs: 6 * 60 * 1000,
queuedOrRetryable: 1,
oldestRunningHeartbeatAgeMs: 16 * 60 * 1000,
});
assert.equal(evaluation.shouldPause, true);
assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_pending_age_ms')));
assert.ok(evaluation.reasons.some((reason) => reason.includes('oldest_running_heartbeat_age_ms')));
assert.equal(now > 0, true);
});
scripts/build-portal-runtime.mjs
+11 -1
View File
@@ -348,10 +348,18 @@ async function writeMetadata() {
path.join(root, 'scripts', 'agent-run-guard.mjs'),
path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'),
);
await fs.copyFile(
path.join(root, 'scripts', 'agent-run-guard-lib.mjs'),
path.join(runtimeRoot, 'scripts', 'agent-run-guard-lib.mjs'),
);
await fs.copyFile(
path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'),
path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'),
);
await fs.copyFile(
path.join(root, 'scripts', 'resume-agent-run-prod.sh'),
path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'),
);
await writeFile(
path.join(runtimeRoot, 'RUNBOOK.txt'),
[
@@ -417,8 +425,9 @@ async function writeMetadata() {
' bash scripts/install-agent-run-worker-agent.sh # installs disabled by default',
' node scripts/check-agent-run-worker.mjs # read-only LaunchAgent/queue check',
' node scripts/agent-run-guard.mjs # dry-run auto-pause guard check',
' node scripts/agent-run-guard.mjs --apply # stop worker and disable code-run gate when thresholds trip',
' node scripts/agent-run-guard.mjs --apply # first-time pause only; repeat checks are no-ops',
' bash scripts/install-agent-run-guard-agent.sh # installs guard LaunchAgent',
' bash scripts/resume-agent-run-prod.sh # manual recovery after guard pause',
'',
].join('\n'),
);
@@ -456,6 +465,7 @@ async function main() {
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'resume-agent-run-prod.sh'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755);
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
scripts/resume-agent-run-prod.sh
Executable
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$ROOT"
HOST="${MEMIND_PROD_HOST:-john@58.38.22.103}"
REMOTE_ROOT="${MEMIND_PROD_ROOT:-/Users/john/Project/Memind}"
echo "[resume-agent-run] stop guard on ${HOST}"
ssh "${HOST}" "launchctl bootout gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true; launchctl disable gui/\$(id -u)/cn.tkmind.memind-agent-run-guard 2>/dev/null || true"
echo "[resume-agent-run] enable code runs + clear stuck queue rows"
ssh "${HOST}" "bash -lc '
set -euo pipefail
cd \"${REMOTE_ROOT}\"
set -a
source .env
set +a
if grep -q \"^MEMIND_AGENT_CODE_RUNS_ENABLED=\" .env; then
sed -i \"\" \"s/^MEMIND_AGENT_CODE_RUNS_ENABLED=.*/MEMIND_AGENT_CODE_RUNS_ENABLED=1/\" .env
else
echo \"MEMIND_AGENT_CODE_RUNS_ENABLED=1\" >> .env
fi
/opt/homebrew/opt/node@24/bin/node <<\"EOF\"
const mysql = require(\"mysql2/promise\");
(async () => {
const pool = mysql.createPool(process.env.DATABASE_URL);
const now = Date.now();
const [runs] = await pool.query(
\"UPDATE h5_agent_runs SET status=?, error_message=COALESCE(error_message, ?), updated_at=?, completed_at=? WHERE status IN (?, ?, ?)\",
[\"failed\", \"resume-agent-run: stale run cleared\", now, now, \"queued\", \"running\", \"retryable\"],
);
const [msgs] = await pool.query(
\"UPDATE h5_wechat_mp_messages SET status=?, updated_at=? WHERE status=?\",
[\"failed\", now, \"processing\"],
);
console.log(JSON.stringify({ agentRunsFailed: runs.affectedRows, wechatMessagesFailed: msgs.affectedRows }));
await pool.end();
})().catch((err) => {
console.error(err);
process.exit(1);
});
EOF
'"
echo "[resume-agent-run] start worker + restart portal once"
ssh "${HOST}" "bash -lc '
GUI=gui/\$(id -u)
launchctl enable \$GUI/cn.tkmind.memind-agent-run-worker 2>/dev/null || true
launchctl bootstrap \$GUI \$HOME/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist 2>/dev/null || true
launchctl kickstart -k \$GUI/cn.tkmind.memind-agent-run-worker || true
launchctl kickstart -k \$GUI/cn.tkmind.memind-portal
sleep 5
curl -s -o /dev/null -w \"portal=%{http_code}\\n\" http://127.0.0.1:8081/api/status
'"
echo "[resume-agent-run] done (guard remains stopped; re-enable manually after queue is healthy)"