feat: recover stale running agent runs

scripts/agent-run-worker.mjs

@@ -36,17 +36,26 @@ function parseArgs(argv) {
   const args = {
     once: false,
     status: false,
+    recoverStale: false,
+    applyRecovery: false,
     runId: '',
     pollMs: Number(process.env.MEMIND_AGENT_RUN_WORKER_POLL_MS ?? 1000),
     limit: Number(process.env.MEMIND_AGENT_RUN_WORKER_BATCH_SIZE ?? 1),
+    staleMs: Number(process.env.MEMIND_AGENT_RUN_STALE_RUNNING_MS ?? process.env.MEMIND_AGENT_RUN_TIMEOUT_MS ?? 15 * 60 * 1000),
   };
   for (let i = 0; i < argv.length; i += 1) {
     const item = argv[i];
     if (item === '--once') args.once = true;
     else if (item === '--status') args.status = true;
+    else if (item === '--recover-stale') args.recoverStale = true;
+    else if (item === '--apply-recovery') {
+      args.recoverStale = true;
+      args.applyRecovery = true;
+    }
     else if (item === '--run-id') args.runId = String(argv[++i] ?? '').trim();
     else if (item === '--poll-ms') args.pollMs = Number(argv[++i] ?? args.pollMs);
     else if (item === '--limit') args.limit = Number(argv[++i] ?? args.limit);
+    else if (item === '--stale-ms') args.staleMs = Number(argv[++i] ?? args.staleMs);
     else if (item === '--help' || item === '-h') args.help = true;
   }
   return args;
@@ -56,9 +65,12 @@ function printHelp() {
   console.log([
     'Usage:',
     '  node scripts/agent-run-worker.mjs [--once] [--status] [--run-id <id>] [--poll-ms 1000] [--limit 1]',
+    '  node scripts/agent-run-worker.mjs --recover-stale [--apply-recovery] [--stale-ms 900000]',
     '',
     'Notes:',
     '  - Processes existing h5_agent_runs queued/retryable rows through the Tool Gateway queue.',
+    '  - Regular dispatch automatically fails stale running rows older than MEMIND_AGENT_RUN_TIMEOUT_MS.',
+    '  - --recover-stale is dry-run by default; --apply-recovery marks stale running rows failed.',
     '  - Does not create agent runs by itself.',
     '  - --run-id dispatches exactly one known run and avoids scanning the global queue.',
     '  - Set MEMIND_AGENT_RUN_AUTODISPATCH=0 on Portal only when this worker is ready to take over.',
@@ -140,6 +152,21 @@ async function runOnce() {
   }));
 }
 
+async function runStaleRecovery() {
+  const result = await gateway.recoverStaleRunningRuns({
+    staleMs: args.staleMs,
+    limit: args.limit,
+    dryRun: !args.applyRecovery,
+    reason: args.applyRecovery ? 'manual_stale_recovery' : 'manual_stale_recovery_dry_run',
+  });
+  console.log(JSON.stringify({
+    ok: true,
+    mode: args.applyRecovery ? 'recover-stale-apply' : 'recover-stale-dry-run',
+    recovery: result,
+    queue: await gateway.getQueueStatus(),
+  }, null, 2));
+}
+
 async function waitForIdle() {
   const startedAt = Date.now();
   const maxWaitMs = Number(process.env.MEMIND_AGENT_RUN_WORKER_DRAIN_WAIT_MS ?? 20 * 60 * 1000);
@@ -157,6 +184,8 @@ let exitCode = 0;
 try {
   if (args.status) {
     console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
+  } else if (args.recoverStale) {
+    await runStaleRecovery();
   } else if (args.once || args.runId) {
     await runOnce();
   } else {

scripts/check-agent-run-worker.mjs

@@ -108,6 +108,15 @@ async function readQueueSummary() {
       ? null
       : Number(lagRows[0].oldest_updated_at);
 
+    const [runningRows] = await conn.query(
+      `SELECT MIN(started_at) AS oldest_started_at
+       FROM h5_agent_runs
+       WHERE status = 'running'`,
+    );
+    const oldestRunningStartedAt = runningRows[0]?.oldest_started_at == null
+      ? null
+      : Number(runningRows[0].oldest_started_at);
+
     const [failedRows] = await conn.query(
       `SELECT id, request_id, error_message, updated_at
        FROM h5_agent_runs
@@ -120,6 +129,8 @@ async function readQueueSummary() {
       statusCounts,
       oldestPendingUpdatedAt: oldestUpdatedAt,
       oldestPendingAgeMs: oldestUpdatedAt == null ? 0 : Math.max(0, Date.now() - oldestUpdatedAt),
+      oldestRunningStartedAt,
+      oldestRunningAgeMs: oldestRunningStartedAt == null ? 0 : Math.max(0, Date.now() - oldestRunningStartedAt),
       latestFailedRun: failedRows[0] ? {
         id: failedRows[0].id,
         requestId: failedRows[0].request_id,