feat: recover stale running agent runs

agent-run-gateway.test.mjs

@@ -33,6 +33,26 @@ function createFakePool() {
         }
         return [[...counts].map(([status, count]) => ({ status, count }))];
       }
+      if (sql.includes('SELECT MIN(started_at) AS oldest_started_at')) {
+        const started = [...runs.values()]
+          .filter((row) => row.status === 'running' && row.started_at != null)
+          .map((row) => Number(row.started_at));
+        return [[{ oldest_started_at: started.length ? Math.min(...started) : null }]];
+      }
+      if (sql.includes('SELECT id, request_id, started_at, updated_at, attempts')) {
+        const [cutoff, limit = 1] = params;
+        return [[...runs.values()]
+          .filter((row) => row.status === 'running' && row.started_at != null && Number(row.started_at) <= Number(cutoff))
+          .sort((a, b) => Number(a.started_at) - Number(b.started_at))
+          .slice(0, Number(limit))
+          .map((row) => ({
+            id: row.id,
+            request_id: row.request_id,
+            started_at: row.started_at,
+            updated_at: row.updated_at,
+            attempts: row.attempts,
+          }))];
+      }
       if (sql.includes('SELECT id') && sql.includes("status IN ('queued', 'retryable')")) {
         const limit = Number(params[0] ?? 1);
         return [[...runs.values()]
@@ -87,6 +107,20 @@ function createFakePool() {
         });
         return [{ affectedRows: 1 }];
       }
+      if (sql.includes("WHERE id = ? AND status = 'running' AND started_at <= ?")) {
+        const [errorMessage, updatedAt, completedAt, id, cutoff] = params;
+        const row = runs.get(id);
+        if (!row || row.status !== 'running' || Number(row.started_at) > Number(cutoff)) {
+          return [{ affectedRows: 0 }];
+        }
+        Object.assign(row, {
+          status: 'failed',
+          error_message: errorMessage,
+          updated_at: updatedAt,
+          completed_at: completedAt,
+        });
+        return [{ affectedRows: 1 }];
+      }
       if (sql.includes('UPDATE h5_agent_runs SET')) {
         const id = params.at(-1);
         const row = runs.get(id);
@@ -610,3 +644,69 @@ test('external worker does not dispatch more runs when local queue is full', asy
   await waitFor(() => pool.runs.get(run1.id)?.status === 'succeeded');
   assert.equal(pool.runs.get(run2.id).status, 'queued');
 });
+
+test('stale running recovery dry-run reports rows without mutating them', async () => {
+  const pool = createFakePool();
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {},
+    autoDispatch: false,
+    runTimeoutMs: 1000,
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-stale-dry-run',
+    userMessage: { role: 'user', content: [] },
+  });
+  Object.assign(pool.runs.get(run.id), {
+    status: 'running',
+    attempts: 1,
+    started_at: Date.now() - 5000,
+    updated_at: Date.now() - 5000,
+  });
+
+  const result = await gateway.recoverStaleRunningRuns({ staleMs: 1000, dryRun: true });
+
+  assert.equal(result.considered, 1);
+  assert.equal(result.recovered, 0);
+  assert.equal(result.runs[0].id, run.id);
+  assert.equal(pool.runs.get(run.id).status, 'running');
+  assert.equal(
+    pool.events.some((event) => event.runId === run.id && event.eventType === 'stale_recovered'),
+    false,
+  );
+});
+
+test('stale running recovery marks old running rows failed with an event', async () => {
+  const pool = createFakePool();
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {},
+    autoDispatch: false,
+    runTimeoutMs: 1000,
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-stale-apply',
+    userMessage: { role: 'user', content: [] },
+  });
+  Object.assign(pool.runs.get(run.id), {
+    status: 'running',
+    attempts: 1,
+    started_at: Date.now() - 5000,
+    updated_at: Date.now() - 5000,
+  });
+
+  const result = await gateway.recoverStaleRunningRuns({ staleMs: 1000, dryRun: false });
+
+  assert.equal(result.considered, 1);
+  assert.equal(result.recovered, 1);
+  assert.equal(pool.runs.get(run.id).status, 'failed');
+  assert.match(pool.runs.get(run.id).error_message, /stale running state/);
+  assert.equal(
+    pool.events.some((event) => event.runId === run.id && event.eventType === 'stale_recovered'),
+    true,
+  );
+});