feat: add worker heartbeat for agent runs

scripts/check-agent-run-worker.mjs

@@ -109,13 +109,43 @@ async function readQueueSummary() {
       : Number(lagRows[0].oldest_updated_at);
 
     const [runningRows] = await conn.query(
-      `SELECT MIN(started_at) AS oldest_started_at
-       FROM h5_agent_runs
-       WHERE status = 'running'`,
+      `SELECT
+         r.id,
+         r.request_id,
+         r.started_at,
+         r.updated_at,
+         h.latest_heartbeat_at
+       FROM h5_agent_runs r
+       LEFT JOIN (
+         SELECT run_id, MAX(created_at) AS latest_heartbeat_at
+         FROM h5_agent_run_events
+         WHERE event_type = 'worker_heartbeat'
+         GROUP BY run_id
+       ) h ON h.run_id = r.id
+       WHERE r.status = 'running'
+       ORDER BY COALESCE(h.latest_heartbeat_at, r.started_at) ASC
+       LIMIT 1`,
     );
-    const oldestRunningStartedAt = runningRows[0]?.oldest_started_at == null
+    const oldestRunningStartedAt = runningRows[0]?.started_at == null
       ? null
-      : Number(runningRows[0].oldest_started_at);
+      : Number(runningRows[0].started_at);
+    const oldestRunningHeartbeatAt = runningRows[0]?.latest_heartbeat_at == null
+      ? null
+      : Number(runningRows[0].latest_heartbeat_at);
+    const oldestRunningHeartbeatAgeMs = runningRows[0]
+      ? Math.max(0, Date.now() - Number(runningRows[0].latest_heartbeat_at ?? runningRows[0].started_at ?? Date.now()))
+      : 0;
+    const [runningWithoutHeartbeatRows] = await conn.query(
+      `SELECT COUNT(*) AS count
+       FROM h5_agent_runs r
+       LEFT JOIN (
+         SELECT run_id, MAX(created_at) AS latest_heartbeat_at
+         FROM h5_agent_run_events
+         WHERE event_type = 'worker_heartbeat'
+         GROUP BY run_id
+       ) h ON h.run_id = r.id
+       WHERE r.status = 'running' AND h.latest_heartbeat_at IS NULL`,
+    );
 
     const [failedRows] = await conn.query(
       `SELECT id, request_id, error_message, updated_at
@@ -131,6 +161,17 @@ async function readQueueSummary() {
       oldestPendingAgeMs: oldestUpdatedAt == null ? 0 : Math.max(0, Date.now() - oldestUpdatedAt),
       oldestRunningStartedAt,
       oldestRunningAgeMs: oldestRunningStartedAt == null ? 0 : Math.max(0, Date.now() - oldestRunningStartedAt),
+      oldestRunningHeartbeatAt,
+      oldestRunningHeartbeatAgeMs,
+      runningWithoutHeartbeatCount: Number(runningWithoutHeartbeatRows[0]?.count ?? 0),
+      latestRunningRun: runningRows[0] ? {
+        id: runningRows[0].id,
+        requestId: runningRows[0].request_id,
+        startedAt: oldestRunningStartedAt,
+        updatedAt: runningRows[0].updated_at == null ? null : Number(runningRows[0].updated_at),
+        heartbeatAt: oldestRunningHeartbeatAt,
+        heartbeatAgeMs: oldestRunningHeartbeatAgeMs,
+      } : null,
       latestFailedRun: failedRows[0] ? {
         id: failedRows[0].id,
         requestId: failedRows[0].request_id,