fix(guard): idempotent agent-run pause to stop portal restart loop

Skip repeat guard pause actions when code runs already disabled; add resume script and tests.
2026-07-04 05:41:51 +00:00
parent 2a3579c73a
commit fafc1fe7fd
5 changed files with 245 additions and 29 deletions
@@ -0,0 +1,77 @@
+export function readEnvValue(envRaw, key) {
+  for (const line of String(envRaw ?? '').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const idx = trimmed.indexOf('=');
+    if (idx < 0) continue;
+    const envKey = trimmed.slice(0, idx).trim();
+    if (envKey !== key) continue;
+    let value = trimmed.slice(idx + 1).trim();
+    if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+      value = value.slice(1, -1);
+    }
+    return value;
+  }
+  return null;
+}
+
+export function isGuardPaused(envRaw) {
+  const value = readEnvValue(envRaw, 'MEMIND_AGENT_CODE_RUNS_ENABLED');
+  return value != null && String(value).trim() === '0';
+}
+
+export function evaluateHealth(queue) {
+  const positiveInt = (value, fallback) => {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
+  };
+  const thresholds = {
+    maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
+    maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
+    maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
+    maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
+  };
+  const reasons = [];
+  if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
+    reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
+  }
+  if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
+    reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
+  }
+  if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
+    reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
+  }
+  if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
+    reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
+  }
+  return { thresholds, reasons, shouldPause: reasons.length > 0 };
+}
+
+export function buildPausePlan({ shouldPause, envRaw, dryRun }) {
+  if (!shouldPause) {
+    return {
+      action: 'none',
+      alreadyPaused: isGuardPaused(envRaw),
+      applied: false,
+      actions: ['skip_pause: queue healthy'],
+    };
+  }
+  if (isGuardPaused(envRaw)) {
+    return {
+      action: 'skip',
+      alreadyPaused: true,
+      applied: false,
+      actions: dryRun
+        ? ['would skip pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0']
+        : ['skip_pause: MEMIND_AGENT_CODE_RUNS_ENABLED already 0'],
+    };
+  }
+  return {
+    action: 'apply',
+    alreadyPaused: false,
+    applied: false,
+    actions: dryRun
+      ? ['would apply first-time pause']
+      : ['apply_pause: first trigger'],
+  };
+}