tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add runtime worker heartbeat sidecar

Browse Source
This commit is contained in:
John
2026-07-02 09:03:30 +08:00
parent a0af049fa7
commit e24e6ca4a7
5 changed files with 277 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/runtime-slo-report.mjs
+14 -2
View File
@@ -225,6 +225,7 @@ async function readRedisSummary(namespace, redisUrl) {
function summarizeRuntime(runtimeJson) {
const workers = runtimeJson?.router?.workers ?? [];
const staleMetricMs = 2 * 60 * 1000;
const staleHeartbeatMs = Number(process.env.MEMIND_RUNTIME_HEARTBEAT_STALE_MS || 60 * 1000);
const now = Date.now();
return {
routerEnabled: Boolean(runtimeJson?.router?.enabled),
@@ -249,6 +250,14 @@ function summarizeRuntime(runtimeJson) {
fdPressure: worker.fdPressure,
fdCount: worker.fdCount,
containerHealth: worker.containerHealth,
heartbeat: worker.heartbeat,
heartbeatAgeMs: worker.heartbeat ? now - worker.heartbeat : null,
heartbeatFresh: worker.heartbeat ? now - worker.heartbeat <= staleHeartbeatMs : false,
heartbeatSource: worker.heartbeatSource ?? null,
heartbeatOk: worker.heartbeatOk ?? null,
heartbeatStatusCode: worker.heartbeatStatusCode ?? null,
heartbeatLatencyMs: worker.heartbeatLatencyMs ?? null,
heartbeatError: worker.heartbeatError ?? null,
metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null,
metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false,
score: worker.score,
@@ -304,6 +313,8 @@ if (!runtime.ok) failures.push('runtime_status_unavailable');
if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled');
for (const worker of runtimeSummary?.workers ?? []) {
if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`);
if (!worker.heartbeatFresh) failures.push(`${worker.id}_heartbeat_stale`);
if (worker.heartbeatOk === false) failures.push(`${worker.id}_heartbeat_unhealthy`);
if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`);
if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`);
if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`);
@@ -361,8 +372,8 @@ function markdownReport(payload) {
'',
'## Workers',
'',
'| worker | healthy | active | errors | first-token 5m p50/p95 | first-token 1h p50/p95 | metricsFresh | score |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
'| worker | healthy | active | errors | first-token 5m p50/p95 | first-token 1h p50/p95 | heartbeat | metricsFresh | score |',
'| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
];
for (const worker of workers) {
lines.push([
@@ -372,6 +383,7 @@ function markdownReport(payload) {
worker.streamErrorCount,
`${worker.firstToken5m?.p50Ms ?? 0}/${worker.firstToken5m?.p95Ms ?? 0}`,
`${worker.firstToken1h?.p50Ms ?? 0}/${worker.firstToken1h?.p95Ms ?? 0}`,
`${worker.heartbeatSource ?? 'none'}:${worker.heartbeatFresh}`,
worker.metricsFresh,
worker.score,
].join(' | ').replace(/^/, '| ').replace(/$/, ' |'));