tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add agent run heartbeat to slo reports

Browse Source
This commit is contained in:
Your Name
2026-07-02 12:41:04 +08:00
parent 054379ac98
commit f9a385f333
5 changed files with 158 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/runtime-slo-report.mjs
+34 -1
View File
@@ -224,6 +224,7 @@ async function readRedisSummary(namespace, redisUrl) {
function summarizeRuntime(runtimeJson) {
const workers = runtimeJson?.router?.workers ?? [];
const toolQueue = runtimeJson?.toolRuntime?.queue ?? null;
const staleMetricMs = 2 * 60 * 1000;
const staleHeartbeatMs = Number(process.env.MEMIND_RUNTIME_HEARTBEAT_STALE_MS || 60 * 1000);
const now = Date.now();
@@ -231,7 +232,20 @@ function summarizeRuntime(runtimeJson) {
routerEnabled: Boolean(runtimeJson?.router?.enabled),
publicBaseUrl: runtimeJson?.publicBaseUrl ?? null,
toolRuntime: runtimeJson?.toolRuntime ?? null,
toolQueue: runtimeJson?.toolRuntime?.queue ?? null,
toolQueue,
toolQueueSlo: toolQueue
? {
heartbeatMs: Number(toolQueue.heartbeatMs ?? 0),
maxConcurrentRuns: Number(toolQueue.maxConcurrentRuns ?? 0),
statusCounts: toolQueue.statusCounts ?? {},
oldestRunningStartedAt: toolQueue.oldestRunningStartedAt ?? null,
oldestRunningAgeMs: Number(toolQueue.oldestRunningAgeMs ?? 0),
oldestRunningHeartbeatAt: toolQueue.oldestRunningHeartbeatAt ?? null,
oldestRunningHeartbeatAgeMs: Number(toolQueue.oldestRunningHeartbeatAgeMs ?? 0),
runningWithoutHeartbeatCount: Number(toolQueue.runningWithoutHeartbeatCount ?? 0),
latestRunningRun: toolQueue.latestRunningRun ?? null,
}
: null,
workers: workers.map((worker) => ({
id: worker.id,
healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null,
@@ -325,6 +339,21 @@ if (runtimeSummary?.toolQueue?.error) failures.push('tool_queue_status_error');
if (runtimeSummary?.toolQueue?.inFlight > runtimeSummary?.toolQueue?.maxConcurrentRuns) {
failures.push('tool_queue_concurrency_exceeded');
}
const agentRunHeartbeatStaleMs = Number(
process.env.MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS ||
Math.max(90 * 1000, Number(runtimeSummary?.toolQueueSlo?.heartbeatMs ?? 0) * 3),
);
if (
runtimeSummary?.toolQueueSlo?.oldestRunningHeartbeatAgeMs > agentRunHeartbeatStaleMs
) {
failures.push(`agent_run_heartbeat_stale_${runtimeSummary.toolQueueSlo.oldestRunningHeartbeatAgeMs}`);
}
if (
Number(runtimeSummary?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0) > 0 &&
Number(runtimeSummary?.toolQueueSlo?.oldestRunningAgeMs ?? 0) > agentRunHeartbeatStaleMs
) {
failures.push(`agent_run_missing_heartbeat_${runtimeSummary.toolQueueSlo.runningWithoutHeartbeatCount}`);
}
const reportDir = path.resolve(args.reportDir || path.join(appRoot, 'reports', 'runtime-slo'));
if (args.writeReport || args.prune) {
@@ -392,6 +421,10 @@ function markdownReport(payload) {
'',
'## Tool Queue',
'',
`- heartbeatMs: ${payload.runtime?.toolQueueSlo?.heartbeatMs ?? 0}`,
`- oldestRunningHeartbeatAgeMs: ${payload.runtime?.toolQueueSlo?.oldestRunningHeartbeatAgeMs ?? 0}`,
`- runningWithoutHeartbeatCount: ${payload.runtime?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0}`,
'',
'```json',
JSON.stringify(payload.runtime?.toolQueue ?? null, null, 2),
'```',