tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add worker heartbeat for agent runs

Browse Source
This commit is contained in:
Your Name
2026-07-02 12:38:34 +08:00
parent 7a47dc6a98
commit 054379ac98
8 changed files with 499 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agent-run-guard.mjs
+48 -7
View File
@@ -141,13 +141,43 @@ async function readQueueHealth(now) {
: Number(oldestPendingRows[0].oldest_updated_at);
const [oldestRunningRows] = await conn.query(
`SELECT MIN(started_at) AS oldest_started_at
FROM h5_agent_runs
WHERE status = 'running'`,
`SELECT
r.id,
r.request_id,
r.started_at,
r.updated_at,
h.latest_heartbeat_at
FROM h5_agent_runs r
LEFT JOIN (
SELECT run_id, MAX(created_at) AS latest_heartbeat_at
FROM h5_agent_run_events
WHERE event_type = 'worker_heartbeat'
GROUP BY run_id
) h ON h.run_id = r.id
WHERE r.status = 'running'
ORDER BY COALESCE(h.latest_heartbeat_at, r.started_at) ASC
LIMIT 1`,
);
const oldestRunningStartedAt = oldestRunningRows[0]?.oldest_started_at == null
const oldestRunningStartedAt = oldestRunningRows[0]?.started_at == null
? null
: Number(oldestRunningRows[0].oldest_started_at);
: Number(oldestRunningRows[0].started_at);
const oldestRunningHeartbeatAt = oldestRunningRows[0]?.latest_heartbeat_at == null
? null
: Number(oldestRunningRows[0].latest_heartbeat_at);
const oldestRunningHeartbeatAgeMs = oldestRunningRows[0]
? Math.max(0, now - Number(oldestRunningRows[0].latest_heartbeat_at ?? oldestRunningRows[0].started_at ?? now))
: 0;
const [runningWithoutHeartbeatRows] = await conn.query(
`SELECT COUNT(*) AS count
FROM h5_agent_runs r
LEFT JOIN (
SELECT run_id, MAX(created_at) AS latest_heartbeat_at
FROM h5_agent_run_events
WHERE event_type = 'worker_heartbeat'
GROUP BY run_id
) h ON h.run_id = r.id
WHERE r.status = 'running' AND h.latest_heartbeat_at IS NULL`,
);
const failedWindowMs = positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_FAILED_WINDOW_MS, 10 * 60 * 1000);
const since = now - failedWindowMs;
@@ -173,6 +203,17 @@ async function readQueueHealth(now) {
oldestPendingAgeMs: oldestPendingUpdatedAt == null ? 0 : Math.max(0, now - oldestPendingUpdatedAt),
oldestRunningStartedAt,
oldestRunningAgeMs: oldestRunningStartedAt == null ? 0 : Math.max(0, now - oldestRunningStartedAt),
oldestRunningHeartbeatAt,
oldestRunningHeartbeatAgeMs,
runningWithoutHeartbeatCount: Number(runningWithoutHeartbeatRows[0]?.count ?? 0),
latestRunningRun: oldestRunningRows[0] ? {
id: oldestRunningRows[0].id,
requestId: oldestRunningRows[0].request_id,
startedAt: oldestRunningStartedAt,
updatedAt: oldestRunningRows[0].updated_at == null ? null : Number(oldestRunningRows[0].updated_at),
heartbeatAt: oldestRunningHeartbeatAt,
heartbeatAgeMs: oldestRunningHeartbeatAgeMs,
} : null,
failedWindowMs,
failedRecentCount: Number(failedRows[0]?.count ?? 0),
latestFailedRun: latestFailedRows[0] ? {
@@ -204,8 +245,8 @@ function evaluateHealth(queue) {
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
}
if (queue.oldestRunningAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_age_ms ${queue.oldestRunningAgeMs} >= ${thresholds.maxRunningAgeMs}`);
if (queue.oldestRunningHeartbeatAgeMs >= thresholds.maxRunningAgeMs) {
reasons.push(`oldest_running_heartbeat_age_ms ${queue.oldestRunningHeartbeatAgeMs} >= ${thresholds.maxRunningAgeMs}`);
}
return { thresholds, reasons, shouldPause: reasons.length > 0 };
}
scripts/check-agent-run-worker.mjs
+46 -5
View File
@@ -109,13 +109,43 @@ async function readQueueSummary() {
: Number(lagRows[0].oldest_updated_at);
const [runningRows] = await conn.query(
`SELECT MIN(started_at) AS oldest_started_at
FROM h5_agent_runs
WHERE status = 'running'`,
`SELECT
r.id,
r.request_id,
r.started_at,
r.updated_at,
h.latest_heartbeat_at
FROM h5_agent_runs r
LEFT JOIN (
SELECT run_id, MAX(created_at) AS latest_heartbeat_at
FROM h5_agent_run_events
WHERE event_type = 'worker_heartbeat'
GROUP BY run_id
) h ON h.run_id = r.id
WHERE r.status = 'running'
ORDER BY COALESCE(h.latest_heartbeat_at, r.started_at) ASC
LIMIT 1`,
);
const oldestRunningStartedAt = runningRows[0]?.oldest_started_at == null
const oldestRunningStartedAt = runningRows[0]?.started_at == null
? null
: Number(runningRows[0].oldest_started_at);
: Number(runningRows[0].started_at);
const oldestRunningHeartbeatAt = runningRows[0]?.latest_heartbeat_at == null
? null
: Number(runningRows[0].latest_heartbeat_at);
const oldestRunningHeartbeatAgeMs = runningRows[0]
? Math.max(0, Date.now() - Number(runningRows[0].latest_heartbeat_at ?? runningRows[0].started_at ?? Date.now()))
: 0;
const [runningWithoutHeartbeatRows] = await conn.query(
`SELECT COUNT(*) AS count
FROM h5_agent_runs r
LEFT JOIN (
SELECT run_id, MAX(created_at) AS latest_heartbeat_at
FROM h5_agent_run_events
WHERE event_type = 'worker_heartbeat'
GROUP BY run_id
) h ON h.run_id = r.id
WHERE r.status = 'running' AND h.latest_heartbeat_at IS NULL`,
);
const [failedRows] = await conn.query(
`SELECT id, request_id, error_message, updated_at
@@ -131,6 +161,17 @@ async function readQueueSummary() {
oldestPendingAgeMs: oldestUpdatedAt == null ? 0 : Math.max(0, Date.now() - oldestUpdatedAt),
oldestRunningStartedAt,
oldestRunningAgeMs: oldestRunningStartedAt == null ? 0 : Math.max(0, Date.now() - oldestRunningStartedAt),
oldestRunningHeartbeatAt,
oldestRunningHeartbeatAgeMs,
runningWithoutHeartbeatCount: Number(runningWithoutHeartbeatRows[0]?.count ?? 0),
latestRunningRun: runningRows[0] ? {
id: runningRows[0].id,
requestId: runningRows[0].request_id,
startedAt: oldestRunningStartedAt,
updatedAt: runningRows[0].updated_at == null ? null : Number(runningRows[0].updated_at),
heartbeatAt: oldestRunningHeartbeatAt,
heartbeatAgeMs: oldestRunningHeartbeatAgeMs,
} : null,
latestFailedRun: failedRows[0] ? {
id: failedRows[0].id,
requestId: failedRows[0].request_id,