From f9a385f333678fb4a5bd232a2bbd82f489f4524d Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 12:41:04 +0800 Subject: [PATCH] feat: add agent run heartbeat to slo reports --- .env.example | 1 + docs/agent-run-worker-rollout-runbook.md | 15 ++++ ...2-runtime-execution-assessment-20260702.md | 45 ++++++++++-- .../memind-2-streaming-agent-runtime-plan.md | 68 +++++++++++++++++++ scripts/runtime-slo-report.mjs | 35 +++++++++- 5 files changed, 158 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 27648ba..9c6f52b 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,7 @@ H5_PUBLIC_BASE_URL=http://127.0.0.1:5173 # MEMIND_AGENT_RUN_WORKER_POLL_MS=1000 # MEMIND_AGENT_RUN_WORKER_BATCH_SIZE=1 # MEMIND_AGENT_RUN_HEARTBEAT_MS=30000 +# MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS=90000 # MEMIND_AGENT_RUN_WORKER_EXPECT_RUNNING=0 # Runtime SLO 日报（默认每日 23:55 写 reports/runtime-slo）。 diff --git a/docs/agent-run-worker-rollout-runbook.md b/docs/agent-run-worker-rollout-runbook.md index d01a1c3..bfcf9a9 100644 --- a/docs/agent-run-worker-rollout-runbook.md +++ b/docs/agent-run-worker-rollout-runbook.md @@ -203,6 +203,21 @@ oldestRunningHeartbeatAgeMs=0 when no running rows runningWithoutHeartbeatCount=0 during normal operation ``` +Runtime SLO report should also expose the same heartbeat summary: + +```bash +node /Users/john/Project/Memind/scripts/runtime-slo-report.mjs +``` + +Expected SLO fields: + +```text +runtime.toolQueueSlo.heartbeatMs=30000 +runtime.toolQueueSlo.oldestRunningHeartbeatAgeMs=0 when no running rows +runtime.toolQueueSlo.runningWithoutHeartbeatCount=0 during normal operation +failures=[] unless heartbeat age exceeds MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS +``` + Apply only when the listed running rows are known stale: ```bash diff --git a/docs/architecture/memind-2-runtime-execution-assessment-20260702.md b/docs/architecture/memind-2-runtime-execution-assessment-20260702.md index ddeb28e..c510ba5 100644 --- a/docs/architecture/memind-2-runtime-execution-assessment-20260702.md +++ b/docs/architecture/memind-2-runtime-execution-assessment-20260702.md @@ -19,12 +19,12 @@ - Stream Controller 已具备 SSE headers、abort propagation、backpressure pipeline。 - Redis Router 已启用，承担 worker runtime state。 - Goose Worker Pool 已从固定单点走向四 worker 可观测调度。 -- Aider/OpenHands 已从普通聊天默认能力中剥离，进入 code mode 和后端灰度门禁；P6.3 已把 code run 从 goosed session extension 外移到 `agent-run-v1` Tool Gateway 协议，P6.4 已完成 Aider 真实执行 canary，P6.5 已完成 OpenHands 真实执行 canary，P6.6 已完成 external worker 精确接管 code-run canary，P6.7 已加入 Tool Gateway 产物校验与输出审计，P6.8 已安装 external worker LaunchAgent，P6.9 已完成带 validation 的 external worker 灰度 canary，P6.10 已加入 external worker 只读观测脚本，P6.11 已加入放量策略门禁，P6.12 已开启全用户长期灰度并通过普通测试用户真实路径，P6.13 已安装自动暂停 guard，P6.15 已让 H5 code-run 自动补 receipt validation 并恢复后端 required validation，P6.17/P6.18 已把 external worker 并发 2 通过 canary 并固化为当前 all-user gray 策略，P6.19 已加入 task-level artifact validation，P8.1 已加入 stale running recovery，P8.2 已完成 Portal DB/Auth 瞬时错误兜底，P8.3 已加入 worker lease heartbeat。 +- Aider/OpenHands 已从普通聊天默认能力中剥离，进入 code mode 和后端灰度门禁；P6.3 已把 code run 从 goosed session extension 外移到 `agent-run-v1` Tool Gateway 协议，P6.4 已完成 Aider 真实执行 canary，P6.5 已完成 OpenHands 真实执行 canary，P6.6 已完成 external worker 精确接管 code-run canary，P6.7 已加入 Tool Gateway 产物校验与输出审计，P6.8 已安装 external worker LaunchAgent，P6.9 已完成带 validation 的 external worker 灰度 canary，P6.10 已加入 external worker 只读观测脚本，P6.11 已加入放量策略门禁，P6.12 已开启全用户长期灰度并通过普通测试用户真实路径，P6.13 已安装自动暂停 guard，P6.15 已让 H5 code-run 自动补 receipt validation 并恢复后端 required validation，P6.17/P6.18 已把 external worker 并发 2 通过 canary 并固化为当前 all-user gray 策略，P6.19 已加入 task-level artifact validation，P8.1 已加入 stale running recovery，P8.2 已完成 Portal DB/Auth 瞬时错误兜底，P8.3 已加入 worker lease heartbeat，P8.4 已把 heartbeat 纳入 runtime SLO 报告。 - PG 和 MindSpace 仍保持生产数据边界，SLO 报告只做统计读取；P6.3-P6.15 不新增 schema migration，不删除或修改既有用户数据。 整体执行评分: 9.95 / 10。 -可以支撑当前 H5 streaming 稳定性改造的基础目标。自动采样、worker sidecar heartbeat、SLO 只读快照、SLO 日报定时器、SLO 日报保留策略、first-token EWMA、first-token p50/p95 窗口趋势、Tool Gateway Queue v0、外部 worker 接管入口、真实 worker canary、后端 code-mode canary、code-run 用户级灰度 gate、H5 页面编辑 UI canary、P6.3 Tool Gateway 协议化、P6.4 Aider 真实 canary、P6.5 OpenHands 真实 canary、P6.6 external worker code-run canary、P6.7 Tool Gateway guardrails、P6.8 worker LaunchAgent、P6.9 validated external worker canary、P6.10 worker observability、P6.11 rollout policy gates、P6.12 all-user gray、P6.13 auto-pause guard、P6.15 H5 validation metadata、P6.17/P6.18 worker concurrency 2、P6.19 task-level artifact validation、P8.1 stale running recovery、P8.2 Portal DB/Auth transient hardening 和 P8.3 worker lease heartbeat 已经落地。P6.22 用户可见进度/失败说明按用户决策暂不开发，后续剩余差距转为更长时间 soak、自动报表告警和更细任务类型的校验策略扩展。 +可以支撑当前 H5 streaming 稳定性改造的基础目标。自动采样、worker sidecar heartbeat、SLO 只读快照、SLO 日报定时器、SLO 日报保留策略、first-token EWMA、first-token p50/p95 窗口趋势、Tool Gateway Queue v0、外部 worker 接管入口、真实 worker canary、后端 code-mode canary、code-run 用户级灰度 gate、H5 页面编辑 UI canary、P6.3 Tool Gateway 协议化、P6.4 Aider 真实 canary、P6.5 OpenHands 真实 canary、P6.6 external worker code-run canary、P6.7 Tool Gateway guardrails、P6.8 worker LaunchAgent、P6.9 validated external worker canary、P6.10 worker observability、P6.11 rollout policy gates、P6.12 all-user gray、P6.13 auto-pause guard、P6.15 H5 validation metadata、P6.17/P6.18 worker concurrency 2、P6.19 task-level artifact validation、P8.1 stale running recovery、P8.2 Portal DB/Auth transient hardening、P8.3 worker lease heartbeat 和 P8.4 heartbeat SLO 已经落地。P6.22 用户可见进度/失败说明按用户决策暂不开发，后续剩余差距转为 24h soak 观察和更细任务类型的校验策略扩展。 ## 实测结果 @@ -1016,6 +1016,42 @@ Data boundary: - worker check `ok=true`, `expected=running` - guard `shouldPause=false` +### P8.4 Long-running Gray Soak and Heartbeat SLO + +结果: 通过，runtime SLO JSON/Markdown 已加入 agent-run heartbeat 摘要和 failure 条件，生产只读与临时报表写入验证均通过。 + +- `runtime.toolQueueSlo` 新增: + - `heartbeatMs` + - `maxConcurrentRuns` + - `statusCounts` + - `oldestRunningStartedAt` + - `oldestRunningAgeMs` + - `oldestRunningHeartbeatAt` + - `oldestRunningHeartbeatAgeMs` + - `runningWithoutHeartbeatCount` + - `latestRunningRun` +- failure 条件新增: + - `agent_run_heartbeat_stale_` + - `agent_run_missing_heartbeat_` +- Markdown Tool Queue 段落新增 heartbeat summary。 +- 测试: + - `node --check scripts/runtime-slo-report.mjs` + - 源码目录只读 SLO `ok=true`, `failures=[]` +- 生产部署: + - backup `/Users/john/Project/memind_backups/20260702-123947-p84-heartbeat-slo` + - deployed `scripts/runtime-slo-report.mjs` + - no service restart +- 生产验证: + - production read-only SLO `ok=true`, `failures=[]` + - `toolQueueSlo.heartbeatMs=30000` + - `toolQueueSlo.statusCounts={}` + - `toolQueueSlo.runningWithoutHeartbeatCount=0` + - temporary JSON/Markdown report files include heartbeat summary fields +- 定时器: + - `cn.tkmind.memind-runtime-slo-report` + - remains scheduled at `23:55` + - writes with `--write-report --prune --retention-days 30` + ## 下一步执行建议 ### P6.22 Task Artifact UX and Failure Messages @@ -1026,6 +1062,5 @@ Data boundary: ### P8.4 Long-running Gray Soak and Heartbeat SLO -- 基于 heartbeat 字段做一轮 24h all-user gray soak。 -- SLO 日报增加 running heartbeat age 和 missing heartbeat count。 -- guard 阈值保守观察，不扩大并发。 +- 第一阶段 SLO 字段已完成。 +- 下一步是 24h all-user gray soak 观察，不扩大并发。 diff --git a/docs/architecture/memind-2-streaming-agent-runtime-plan.md b/docs/architecture/memind-2-streaming-agent-runtime-plan.md index 2346055..126a211 100644 --- a/docs/architecture/memind-2-streaming-agent-runtime-plan.md +++ b/docs/architecture/memind-2-streaming-agent-runtime-plan.md @@ -30,6 +30,7 @@ - P8.1 Queue Lease / Stuck Run Recovery: 已完成第一步，worker dispatch 前自动回收超时 running run，新增 `--recover-stale` dry-run/apply 运维入口，生产 synthetic stale run 验证通过。 - P8.2 Portal DB/Auth Transient Error Hardening: 已完成第一步，session attach、`/auth/status` 和 API auth middleware 已捕获 DB/auth 瞬时错误，生产部署后 live health、auth/status、runtime/status、guard 和 SLO 验证通过。 - P8.3 Worker Lease Heartbeat: 已完成第一步，running run 执行期间写入 `worker_heartbeat` event，runtime/status、worker check、guard 和 stale recovery 均优先使用 heartbeat age，生产 synthetic 与真实 worker canary 验证通过。 +- P8.4 Long-running Gray Soak and Heartbeat SLO: 已完成第一步，runtime SLO JSON/Markdown 报告加入 agent-run heartbeat 摘要和 failure 条件，生产只读与临时报表写入验证通过；继续保持并发 2 soak，不扩大灰度。 - P5.15 Active Stream TTL Reconcile: 已按用户要求跳过，暂不做报表/定时 reconcile。 - P5 Worker Pool 运维化: 已完成第一步，Redis Router 支持 worker drain。 - P5.9 First-token Latency EWMA: 已完成，StreamController 会把首个 SSE chunk 延迟写入 Redis，SLO 报告已展示。 @@ -3015,6 +3016,73 @@ runtime/status: - heartbeat data includes `pid=31341`, `attempt=1`, `heartbeatMs=30000` - `tool_gateway_validation` passed for receipt and `public/p83-heartbeat-real-20260702043642.html` +### 2026-07-02 P8.4 Long-running Gray Soak and Heartbeat SLO + +目标: + +- 不扩大并发、不做 UI，先把 P8.3 heartbeat 纳入每日 SLO 报告。 +- 让 23:55 定时日报和手动 `runtime-slo-report.mjs` 都能暴露 running heartbeat 风险。 +- 为后续 24h all-user gray soak 提供稳定观测字段。 + +改动: + +- `scripts/runtime-slo-report.mjs`: + - `runtime.toolQueueSlo` 新增 heartbeat 摘要: + - `heartbeatMs` + - `maxConcurrentRuns` + - `statusCounts` + - `oldestRunningStartedAt` + - `oldestRunningAgeMs` + - `oldestRunningHeartbeatAt` + - `oldestRunningHeartbeatAgeMs` + - `runningWithoutHeartbeatCount` + - `latestRunningRun` + - failure 条件新增: + - `agent_run_heartbeat_stale_`: running heartbeat age 超过阈值。 + - `agent_run_missing_heartbeat_`: running run 长时间缺失 heartbeat。 + - Markdown 报告的 Tool Queue 段落展示: + - `heartbeatMs` + - `oldestRunningHeartbeatAgeMs` + - `runningWithoutHeartbeatCount` +- `.env.example`: + - 记录 `MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS=90000`。 +- `docs/agent-run-worker-rollout-runbook.md`: + - 记录 heartbeat SLO 检查命令和期望字段。 + +测试: + +- `node --check scripts/runtime-slo-report.mjs` 通过。 +- 源码目录只读 SLO: + - `ok=true` + - `failures=[]` + - `toolQueueSlo.heartbeatMs=30000` + - `toolQueueSlo.runningWithoutHeartbeatCount=0` + +生产部署: + +- 备份: + - `/Users/john/Project/memind_backups/20260702-123947-p84-heartbeat-slo` +- 已部署: + - `scripts/runtime-slo-report.mjs` +- 未重启 Portal、worker、goosed 或 H5。 +- 现有 SLO LaunchAgent 保持: + - `cn.tkmind.memind-runtime-slo-report` + - 每日 `23:55` + - `--write-report --prune --retention-days 30` + +生产验证: + +- 生产只读 SLO: + - `ok=true` + - `failures=[]` + - `toolQueueSlo.heartbeatMs=30000` + - `toolQueueSlo.statusCounts={}` + - `toolQueueSlo.oldestRunningHeartbeatAgeMs=0` + - `toolQueueSlo.runningWithoutHeartbeatCount=0` +- 临时 report-dir 写入验证: + - JSON 和 Markdown 均生成成功。 + - 两种格式均包含 `heartbeatMs`、`oldestRunningHeartbeatAgeMs`、`runningWithoutHeartbeatCount`。 + ## 回滚策略 - P0: 修改前保留 `server.mjs` 备份；如启动失败，恢复备份并 `launchctl kickstart` Portal。 diff --git a/scripts/runtime-slo-report.mjs b/scripts/runtime-slo-report.mjs index 6c26218..4d66980 100755 --- a/scripts/runtime-slo-report.mjs +++ b/scripts/runtime-slo-report.mjs @@ -224,6 +224,7 @@ async function readRedisSummary(namespace, redisUrl) { function summarizeRuntime(runtimeJson) { const workers = runtimeJson?.router?.workers ?? []; + const toolQueue = runtimeJson?.toolRuntime?.queue ?? null; const staleMetricMs = 2 * 60 * 1000; const staleHeartbeatMs = Number(process.env.MEMIND_RUNTIME_HEARTBEAT_STALE_MS || 60 * 1000); const now = Date.now(); @@ -231,7 +232,20 @@ function summarizeRuntime(runtimeJson) { routerEnabled: Boolean(runtimeJson?.router?.enabled), publicBaseUrl: runtimeJson?.publicBaseUrl ?? null, toolRuntime: runtimeJson?.toolRuntime ?? null, - toolQueue: runtimeJson?.toolRuntime?.queue ?? null, + toolQueue, + toolQueueSlo: toolQueue + ? { + heartbeatMs: Number(toolQueue.heartbeatMs ?? 0), + maxConcurrentRuns: Number(toolQueue.maxConcurrentRuns ?? 0), + statusCounts: toolQueue.statusCounts ?? {}, + oldestRunningStartedAt: toolQueue.oldestRunningStartedAt ?? null, + oldestRunningAgeMs: Number(toolQueue.oldestRunningAgeMs ?? 0), + oldestRunningHeartbeatAt: toolQueue.oldestRunningHeartbeatAt ?? null, + oldestRunningHeartbeatAgeMs: Number(toolQueue.oldestRunningHeartbeatAgeMs ?? 0), + runningWithoutHeartbeatCount: Number(toolQueue.runningWithoutHeartbeatCount ?? 0), + latestRunningRun: toolQueue.latestRunningRun ?? null, + } + : null, workers: workers.map((worker) => ({ id: worker.id, healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null, @@ -325,6 +339,21 @@ if (runtimeSummary?.toolQueue?.error) failures.push('tool_queue_status_error'); if (runtimeSummary?.toolQueue?.inFlight > runtimeSummary?.toolQueue?.maxConcurrentRuns) { failures.push('tool_queue_concurrency_exceeded'); } +const agentRunHeartbeatStaleMs = Number( + process.env.MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS || + Math.max(90 * 1000, Number(runtimeSummary?.toolQueueSlo?.heartbeatMs ?? 0) * 3), +); +if ( + runtimeSummary?.toolQueueSlo?.oldestRunningHeartbeatAgeMs > agentRunHeartbeatStaleMs +) { + failures.push(`agent_run_heartbeat_stale_${runtimeSummary.toolQueueSlo.oldestRunningHeartbeatAgeMs}`); +} +if ( + Number(runtimeSummary?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0) > 0 && + Number(runtimeSummary?.toolQueueSlo?.oldestRunningAgeMs ?? 0) > agentRunHeartbeatStaleMs +) { + failures.push(`agent_run_missing_heartbeat_${runtimeSummary.toolQueueSlo.runningWithoutHeartbeatCount}`); +} const reportDir = path.resolve(args.reportDir || path.join(appRoot, 'reports', 'runtime-slo')); if (args.writeReport || args.prune) { @@ -392,6 +421,10 @@ function markdownReport(payload) { '', '## Tool Queue', '', + `- heartbeatMs: ${payload.runtime?.toolQueueSlo?.heartbeatMs ?? 0}`, + `- oldestRunningHeartbeatAgeMs: ${payload.runtime?.toolQueueSlo?.oldestRunningHeartbeatAgeMs ?? 0}`, + `- runningWithoutHeartbeatCount: ${payload.runtime?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0}`, + '', '```json', JSON.stringify(payload.runtime?.toolQueue ?? null, null, 2), '```',