tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(guard): idempotent agent-run pause to stop portal restart loop

Browse Source 
Skip repeat guard pause actions when code runs already disabled; add resume script and tests.
This commit was merged in pull request #3.
This commit is contained in:
tkmind
2026-07-04 05:41:51 +00:00
parent 2a3579c73a
commit fafc1fe7fd
5 changed files with 245 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agent-run-guard.mjs
+29 -28
View File
@@ -4,6 +4,7 @@ import path from 'node:path';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import mysql from 'mysql2/promise';
import { buildPausePlan, evaluateHealth } from './agent-run-guard-lib.mjs';
const execFileAsync = promisify(execFile);
const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker';
@@ -49,7 +50,11 @@ function printHelp() {
' node scripts/agent-run-guard.mjs [--dry-run]',
' node scripts/agent-run-guard.mjs --apply',
'',
'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.',
'Dry-run is the default. --apply stops the external worker and disables code-run gate in .env',
'on the first pause trigger only. Repeat checks while already paused are no-ops.',
'',
'Recovery after manual intervention:',
' bash scripts/resume-agent-run-prod.sh',
].join('\n'));
}
@@ -228,29 +233,6 @@ async function readQueueHealth(now) {
}
}
function evaluateHealth(queue) {
const thresholds = {
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
};
const reasons = [];
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
}
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
}
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) {
const actions = [];
const updates = {
@@ -304,6 +286,7 @@ if (args.help) {
const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env');
loadEnvFile(envFile);
const envRaw = fs.existsSync(envFile) ? fs.readFileSync(envFile, 'utf8') : '';
const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null
? true
@@ -318,10 +301,20 @@ const queue = await readQueueHealth(now).catch((err) => ({
const evaluation = queue.error
? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false }
: evaluateHealth(queue);
let pause = { applied: false, actions: [] };
const pausePlan = buildPausePlan({
shouldPause: evaluation.shouldPause,
envRaw,
dryRun: args.dryRun,
});
let pause = {
applied: false,
skipped: pausePlan.action === 'skip',
alreadyPaused: pausePlan.alreadyPaused,
actions: [...pausePlan.actions],
};
if (enabled && evaluation.shouldPause) {
pause = await disableCodeRuns({
if (enabled && pausePlan.action === 'apply') {
const applied = await disableCodeRuns({
root,
envFile,
workerLabel,
@@ -330,10 +323,18 @@ if (enabled && evaluation.shouldPause) {
reasons: evaluation.reasons,
dryRun: args.dryRun,
});
pause = {
...pause,
...applied,
skipped: false,
alreadyPaused: false,
};
}
const result = {
ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true,
ok: enabled
? !evaluation.shouldPause || pause.applied || pause.skipped || args.dryRun
: true,
checkedAt: new Date(now).toISOString(),
enabled,
mode: args.apply ? 'apply' : 'dry-run',