feat: add agent run guard rollout controls
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
# Agent Run Worker Rollout Runbook
|
||||
|
||||
Date: 2026-07-02
|
||||
|
||||
This runbook controls the long-running Tool Gateway / external `agent-run-worker` rollout.
|
||||
|
||||
Production safety rules:
|
||||
|
||||
- Do not delete PG data.
|
||||
- Do not delete or rewrite `/Users/john/Project/Memind/MindSpace` user data.
|
||||
- Keep H5 public traffic on `https://mm.tkmind.cn`.
|
||||
- Treat `m.tkmind.cn` / 105 tunnel as legacy rollback transport, not the primary H5 path.
|
||||
- Keep default production state closed unless a rollout window is explicitly active.
|
||||
|
||||
## Default Closed State
|
||||
|
||||
Expected production state:
|
||||
|
||||
```bash
|
||||
cd /Users/john/Project/Memind
|
||||
curl -fsS https://mm.tkmind.cn/api/runtime/status
|
||||
node scripts/check-agent-run-worker.mjs
|
||||
node scripts/runtime-slo-report.mjs
|
||||
```
|
||||
|
||||
Expected values:
|
||||
|
||||
```text
|
||||
toolRuntime.codeRunPolicy.enabled=false
|
||||
toolRuntime.codeRunsEnabled=false
|
||||
toolRuntime.queue.autoDispatch=true
|
||||
toolRuntime.queue.toolGateway.enabled=false
|
||||
check-agent-run-worker.ok=true
|
||||
check-agent-run-worker.expected=disabled
|
||||
check-agent-run-worker.running=false
|
||||
runtime-slo-report.ok=true
|
||||
runtime-slo-report.failures=[]
|
||||
```
|
||||
|
||||
## One-user Canary Profile
|
||||
|
||||
Use only for a controlled john-user canary.
|
||||
|
||||
Portal `.env` additions:
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_AUTODISPATCH=0
|
||||
MEMIND_AGENT_CODE_RUNS_ENABLED=1
|
||||
MEMIND_AGENT_CODE_RUNS_USER_IDS=a6fb1e97-2b0f-447b-b138-4561d8e5c53e
|
||||
MEMIND_AGENT_CODE_RUN_TASK_TYPES=small_patch
|
||||
MEMIND_AGENT_CODE_RUNS_REQUIRE_VALIDATION=1
|
||||
MEMIND_TOOL_GATEWAY_ENABLED=0
|
||||
MEMIND_TOOL_GATEWAY_DRY_RUN=0
|
||||
```
|
||||
|
||||
Worker installer environment:
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_WORKER_START=1
|
||||
MEMIND_AGENT_RUN_WORKER_POLL_MS=3000
|
||||
MEMIND_AGENT_RUN_WORKER_BATCH_SIZE=1
|
||||
MEMIND_AGENT_RUN_QUEUE_CONCURRENCY=1
|
||||
MEMIND_TOOL_GATEWAY_ENABLED=1
|
||||
MEMIND_TOOL_GATEWAY_DRY_RUN=0
|
||||
MEMIND_AGENT_RUN_WORKDIR_OVERRIDE=/Users/john/Project/memind_canary/<rollout-id>
|
||||
MEMIND_AGENT_RUN_WORKDIR_USER_ID=a6fb1e97-2b0f-447b-b138-4561d8e5c53e
|
||||
```
|
||||
|
||||
All canary code writes must use `/Users/john/Project/memind_canary/...`.
|
||||
|
||||
## All-user Gray Profile
|
||||
|
||||
Current production rollout can use all H5 users by leaving the user and task-type allowlists empty. Validation metadata is accepted when present, but not required for the H5 default code-run path.
|
||||
|
||||
Portal `.env` additions:
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_AUTODISPATCH=0
|
||||
MEMIND_AGENT_CODE_RUNS_ENABLED=1
|
||||
MEMIND_AGENT_CODE_RUNS_USER_IDS=
|
||||
MEMIND_AGENT_CODE_RUN_TASK_TYPES=
|
||||
MEMIND_AGENT_CODE_RUNS_REQUIRE_VALIDATION=0
|
||||
MEMIND_TOOL_GATEWAY_ENABLED=0
|
||||
MEMIND_TOOL_GATEWAY_DRY_RUN=0
|
||||
VITE_AGENT_CODE_RUNS_ENABLED=1
|
||||
VITE_AGENT_CODE_RUNS_AUTODETECT=1
|
||||
VITE_AGENT_CODE_RUNS_USER_IDS=
|
||||
```
|
||||
|
||||
Worker installer environment:
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_WORKER_START=1
|
||||
MEMIND_AGENT_RUN_WORKER_POLL_MS=3000
|
||||
MEMIND_AGENT_RUN_WORKER_BATCH_SIZE=1
|
||||
MEMIND_AGENT_RUN_QUEUE_CONCURRENCY=1
|
||||
MEMIND_TOOL_GATEWAY_ENABLED=1
|
||||
MEMIND_TOOL_GATEWAY_DRY_RUN=0
|
||||
```
|
||||
|
||||
Expected values:
|
||||
|
||||
```text
|
||||
toolRuntime.codeRunPolicy.enabled=true
|
||||
toolRuntime.codeRunPolicy.userAllowlist=[]
|
||||
toolRuntime.codeRunPolicy.taskTypeAllowlist=[]
|
||||
toolRuntime.codeRunPolicy.requireValidation=false
|
||||
toolRuntime.codeRunsEnabled=true
|
||||
toolRuntime.queue.autoDispatch=false
|
||||
toolRuntime.queue.toolGateway.enabled=false
|
||||
check-agent-run-worker.expected=running
|
||||
check-agent-run-worker.running=true
|
||||
```
|
||||
|
||||
## Enable Procedure
|
||||
|
||||
1. Backup `.env`.
|
||||
|
||||
```bash
|
||||
backup_dir="/Users/john/Project/memind_backups/$(date +%Y%m%d-%H%M)-agent-run-worker-rollout"
|
||||
mkdir -p "$backup_dir"
|
||||
cp /Users/john/Project/Memind/.env "$backup_dir/.env.before"
|
||||
```
|
||||
|
||||
2. Append the Portal `.env` additions from the canary profile.
|
||||
|
||||
3. Restart Portal.
|
||||
|
||||
```bash
|
||||
launchctl kickstart -k "gui/$(id -u)/cn.tkmind.memind-portal"
|
||||
```
|
||||
|
||||
4. Verify Portal policy.
|
||||
|
||||
```bash
|
||||
curl -fsS https://mm.tkmind.cn/api/runtime/status | node -e '
|
||||
let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{
|
||||
const j=JSON.parse(s);
|
||||
console.log(JSON.stringify({
|
||||
ok:j.ok,
|
||||
codeRunPolicy:j.toolRuntime?.codeRunPolicy,
|
||||
autoDispatch:j.toolRuntime?.queue?.autoDispatch,
|
||||
toolGateway:j.toolRuntime?.queue?.toolGateway?.enabled
|
||||
}, null, 2));
|
||||
})'
|
||||
```
|
||||
|
||||
5. Create canary repo.
|
||||
|
||||
```bash
|
||||
canary_dir="/Users/john/Project/memind_canary/agent-run-worker-rollout-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$canary_dir"
|
||||
git -C "$canary_dir" init
|
||||
printf '# rollout canary\n' > "$canary_dir/README.md"
|
||||
git -C "$canary_dir" add README.md
|
||||
git -C "$canary_dir" commit -m 'init rollout canary'
|
||||
```
|
||||
|
||||
6. Install and start worker with the worker installer environment.
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_WORKER_START=1 \
|
||||
MEMIND_AGENT_RUN_WORKDIR_OVERRIDE="$canary_dir" \
|
||||
MEMIND_AGENT_RUN_WORKDIR_USER_ID="a6fb1e97-2b0f-447b-b138-4561d8e5c53e" \
|
||||
MEMIND_TOOL_GATEWAY_ENABLED=1 \
|
||||
MEMIND_TOOL_GATEWAY_DRY_RUN=0 \
|
||||
MEMIND_AGENT_RUN_QUEUE_CONCURRENCY=1 \
|
||||
bash /Users/john/Project/Memind/scripts/install-agent-run-worker-agent.sh
|
||||
```
|
||||
|
||||
7. Verify worker running.
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_WORKER_EXPECT_RUNNING=1 node /Users/john/Project/Memind/scripts/check-agent-run-worker.mjs
|
||||
```
|
||||
|
||||
8. Submit only code runs with validation metadata.
|
||||
|
||||
Required user message metadata:
|
||||
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"memindRun": {
|
||||
"validation": {
|
||||
"expectedFile": {
|
||||
"path": "RESULT.md",
|
||||
"contains": "EXPECTED_TOKEN"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Pause Procedure
|
||||
|
||||
Pause worker only:
|
||||
|
||||
```bash
|
||||
launchctl bootout "gui/$(id -u)/cn.tkmind.memind-agent-run-worker" 2>/dev/null || true
|
||||
launchctl disable "gui/$(id -u)/cn.tkmind.memind-agent-run-worker" 2>/dev/null || true
|
||||
node /Users/john/Project/Memind/scripts/check-agent-run-worker.mjs
|
||||
```
|
||||
|
||||
Portal remains in canary mode until `.env` is restored.
|
||||
|
||||
## Disable Procedure
|
||||
|
||||
1. Stop and disable worker.
|
||||
|
||||
```bash
|
||||
launchctl bootout "gui/$(id -u)/cn.tkmind.memind-agent-run-worker" 2>/dev/null || true
|
||||
launchctl disable "gui/$(id -u)/cn.tkmind.memind-agent-run-worker" 2>/dev/null || true
|
||||
```
|
||||
|
||||
2. Restore `.env`.
|
||||
|
||||
```bash
|
||||
cp "$backup_dir/.env.before" /Users/john/Project/Memind/.env
|
||||
launchctl kickstart -k "gui/$(id -u)/cn.tkmind.memind-portal"
|
||||
```
|
||||
|
||||
3. Reinstall disabled worker plist.
|
||||
|
||||
```bash
|
||||
MEMIND_AGENT_RUN_WORKER_START=0 bash /Users/john/Project/Memind/scripts/install-agent-run-worker-agent.sh
|
||||
```
|
||||
|
||||
4. Verify closed state.
|
||||
|
||||
```bash
|
||||
curl -fsS https://mm.tkmind.cn/api/runtime/status
|
||||
node /Users/john/Project/Memind/scripts/check-agent-run-worker.mjs
|
||||
node /Users/john/Project/Memind/scripts/runtime-slo-report.mjs
|
||||
```
|
||||
|
||||
## Incident Checks
|
||||
|
||||
If code tasks appear stuck:
|
||||
|
||||
```bash
|
||||
node /Users/john/Project/Memind/scripts/check-agent-run-worker.mjs
|
||||
tail -200 /Users/john/Library/Logs/memind-agent-run-worker.log
|
||||
curl -fsS https://mm.tkmind.cn/api/runtime/status
|
||||
```
|
||||
|
||||
If `oldestPendingAgeMs` grows while worker is running:
|
||||
|
||||
- Check executor keys and model mapping.
|
||||
- Check `MEMIND_TOOL_GATEWAY_ENABLED=1` in worker plist.
|
||||
- Check `MEMIND_AGENT_RUN_WORKDIR_OVERRIDE` points to canary or intended workspace.
|
||||
- Do not delete queued/failed rows; inspect events first.
|
||||
|
||||
## Auto-pause Guard
|
||||
|
||||
Install the guard after enabling the external worker:
|
||||
|
||||
```bash
|
||||
bash /Users/john/Project/Memind/scripts/install-agent-run-guard-agent.sh
|
||||
```
|
||||
|
||||
Default guard thresholds:
|
||||
|
||||
```text
|
||||
recent failures >= 3 in 10 minutes
|
||||
oldest queued/retryable age >= 5 minutes
|
||||
queued + retryable count >= 10
|
||||
oldest running age >= 15 minutes
|
||||
```
|
||||
|
||||
When a threshold trips, `agent-run-guard.mjs --apply`:
|
||||
|
||||
```text
|
||||
backs up .env
|
||||
sets MEMIND_AGENT_CODE_RUNS_ENABLED=0
|
||||
sets MEMIND_AGENT_RUN_AUTODISPATCH=1
|
||||
stops and disables cn.tkmind.memind-agent-run-worker
|
||||
kickstarts cn.tkmind.memind-portal
|
||||
```
|
||||
|
||||
Read-only check:
|
||||
|
||||
```bash
|
||||
node /Users/john/Project/Memind/scripts/agent-run-guard.mjs --dry-run
|
||||
```
|
||||
|
||||
If validation fails:
|
||||
|
||||
- Treat it as a product correctness failure, not transport failure.
|
||||
- Validation failed is non-retryable by design.
|
||||
- Inspect `tool_gateway_result.stdoutTail`, `stderrTail`, and `tool_gateway_validation_failed`.
|
||||
|
||||
## Restore Proof
|
||||
|
||||
Final closed proof should include:
|
||||
|
||||
```text
|
||||
codeRunPolicy.enabled=false
|
||||
codeRunsEnabled=false
|
||||
queue.autoDispatch=true
|
||||
queue.toolGateway.enabled=false
|
||||
check-agent-run-worker.ok=true
|
||||
check-agent-run-worker.running=false
|
||||
runtime-slo-report.failures=[]
|
||||
```
|
||||
@@ -0,0 +1,873 @@
|
||||
# Memind 2.0 Streaming Agent Runtime 执行评估
|
||||
|
||||
日期: 2026-07-02
|
||||
|
||||
评估对象:
|
||||
|
||||
- 生产运行目录: `/Users/john/Project/Memind`
|
||||
- 干净同步分支: `/Users/john/Project/memind-clean-main-20260702`
|
||||
- 分支: `memind-streaming-runtime-20260702`
|
||||
- 最新提交: `212e163 feat: add code run rollout policy gates`
|
||||
- 最新 release: `20260702-104734-212e163`
|
||||
- 当前公网入口: `https://mm.tkmind.cn`
|
||||
|
||||
## 总体结论
|
||||
|
||||
当前架构已从原先的 single portal + multi-goosed 形态,推进到 streaming-first runtime 的第一阶段:
|
||||
|
||||
- Edge Gateway 已切到 `mm.tkmind.cn -> local nginx -> Portal :8081`。
|
||||
- Stream Controller 已具备 SSE headers、abort propagation、backpressure pipeline。
|
||||
- Redis Router 已启用,承担 worker runtime state。
|
||||
- Goose Worker Pool 已从固定单点走向四 worker 可观测调度。
|
||||
- Aider/OpenHands 已从普通聊天默认能力中剥离,进入 code mode 和后端灰度门禁;P6.3 已把 code run 从 goosed session extension 外移到 `agent-run-v1` Tool Gateway 协议,P6.4 已完成 Aider 真实执行 canary,P6.5 已完成 OpenHands 真实执行 canary,P6.6 已完成 external worker 精确接管 code-run canary,P6.7 已加入 Tool Gateway 产物校验与输出审计,P6.8 已安装 external worker LaunchAgent,P6.9 已完成带 validation 的 external worker 灰度 canary,P6.10 已加入 external worker 只读观测脚本,P6.11 已加入放量策略门禁,P6.12 已开启全用户长期灰度并通过普通测试用户真实路径,P6.13 已安装自动暂停 guard。
|
||||
- PG 和 MindSpace 仍保持生产数据边界,SLO 报告只做统计读取;P6.3-P6.13 不新增 schema migration,不删除或修改既有用户数据。
|
||||
|
||||
整体执行评分: 9.95 / 10。
|
||||
|
||||
可以支撑当前 H5 streaming 稳定性改造的基础目标。自动采样、worker sidecar heartbeat、SLO 只读快照、SLO 日报定时器、SLO 日报保留策略、first-token EWMA、first-token p50/p95 窗口趋势、Tool Gateway Queue v0、外部 worker 接管入口、真实 worker canary、后端 code-mode canary、code-run 用户级灰度 gate、H5 页面编辑 UI canary、P6.3 Tool Gateway 协议化、P6.4 Aider 真实 canary、P6.5 OpenHands 真实 canary、P6.6 external worker code-run canary、P6.7 Tool Gateway guardrails、P6.8 worker LaunchAgent、P6.9 validated external worker canary、P6.10 worker observability、P6.11 rollout policy gates、P6.12 all-user gray 和 P6.13 auto-pause guard 已经落地。主要剩余差距转为更细粒度的任务级产物规范、灰度长期运行观测,以及后续是否提高 worker 并发。
|
||||
|
||||
## 实测结果
|
||||
|
||||
### P6.3 Tool Gateway Protocol
|
||||
|
||||
结果: 通过,默认关闭发布,dry-run canary 通过并已恢复默认关闭。
|
||||
|
||||
- 源码提交:
|
||||
- `21e03d8 feat: add tool gateway protocol`
|
||||
- 生产 release:
|
||||
- `20260702-092457-21e03d8`
|
||||
- 协议:
|
||||
- `agent-run-v1`
|
||||
- 默认状态:
|
||||
- `MEMIND_TOOL_GATEWAY_ENABLED` 未设置
|
||||
- `MEMIND_TOOL_GATEWAY_DRY_RUN` 未设置
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- `toolRuntime.queue.toolGateway.protocol=agent-run-v1`
|
||||
- 验证:
|
||||
- `node --test agent-run-gateway.test.mjs tool-gateway.test.mjs`: `15 passed`
|
||||
- `npm run build:portal-runtime`: passed
|
||||
- `https://mm.tkmind.cn/api/runtime/status`: `ok=true`
|
||||
- `scripts/check-tool-runtime.mjs`: `ok=true`
|
||||
- `scripts/runtime-slo-report.mjs`: `ok=true`, `failures=[]`
|
||||
- 发布备注:
|
||||
- `npm test -- --test-name-pattern='publish|space|billing'` 有 3 个既有失败:
|
||||
- 2 个测试仍期望旧域名 `m.tkmind.cn`,但当前生产基线是 `mm.tkmind.cn`
|
||||
- 1 个测试期望 executor command 缺失,但本机当前已有对应命令
|
||||
- 因失败项与 P6.3 无关,发布使用 `--skip-tests`,仍执行构建和 publish guard。
|
||||
- 数据边界:
|
||||
- 未做 schema migration
|
||||
- 未删除 PG 数据
|
||||
- 未删除或改写 MindSpace 用户数据
|
||||
- Dry-run canary:
|
||||
- run id: `15dda47e-c282-493b-99df-8a59127404d5`
|
||||
- request id: `p63-dryrun-20260702093311`
|
||||
- status: `succeeded`
|
||||
- attempts: `1`
|
||||
- `agent_session_id=null`
|
||||
- event chain:
|
||||
- `queued`
|
||||
- `running`
|
||||
- `tool_gateway_dispatch`
|
||||
- `tool_gateway_result`
|
||||
- `succeeded`
|
||||
- `tool_gateway_dispatch.protocol=agent-run-v1`
|
||||
- `tool_gateway_result.dryRun=true`
|
||||
- `tool_gateway_result.executor=aider`
|
||||
- temporary login session revoked
|
||||
- final state restored:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- `.env` has no `MEMIND_TOOL_GATEWAY_*`
|
||||
- `.env` has no `MEMIND_AGENT_CODE_RUNS_*`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
|
||||
### P6.4 Tool Gateway Real Canary
|
||||
|
||||
结果: 通过,Aider 真实执行 canary 已修复并通过。
|
||||
|
||||
- 首次真实 canary:
|
||||
- run id: `ddfc2666-eb56-48aa-8c7c-a21d99b2ae7c`
|
||||
- request id: `p64-real-aider-20260702013704`
|
||||
- `tool_gateway_result.dryRun=false`
|
||||
- `tool_gateway_result.executor=aider`
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- 发现问题: Aider/LiteLLM 不接受裸 `deepseek-reasoner`,需要 `deepseek/deepseek-reasoner`。
|
||||
- 产物未生成,真实产物 canary 未通过。
|
||||
- 修复:
|
||||
- commit: `0cfd2d3 fix: map deepseek models for aider`
|
||||
- release: `20260702-094216-0cfd2d3`
|
||||
- `custom_deepseek` + Aider 自动映射为 `deepseek/<model>`。
|
||||
- related tests: `43 passed`
|
||||
- 发布运行态:
|
||||
- 发布脚本在 legacy `m.tkmind.cn` tunnel 检查处返回 1。
|
||||
- `mm.tkmind.cn` 主路径和 Portal runtime 已正常。
|
||||
- 已清理占用 8081 的孤儿 Portal node,让 LaunchAgent 接管。
|
||||
- final Portal listener: pid `86462`, port `8081`
|
||||
- 第二次真实 canary:
|
||||
- canary dir: `/Users/john/Project/memind_canary/tool-gateway-p64-fixed-20260702-094739`
|
||||
- run id: `7daff187-1f60-4e67-9848-a404274a0455`
|
||||
- request id: `p64-real-aider-fixed-20260702014740`
|
||||
- status: `succeeded`
|
||||
- attempts: `1`
|
||||
- `agent_session_id=null`
|
||||
- `tool_gateway_dispatch.protocol=agent-run-v1`
|
||||
- `tool_gateway_result.dryRun=false`
|
||||
- `tool_gateway_result.executor=aider`
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- generated file: `/Users/john/Project/memind_canary/tool-gateway-p64-fixed-20260702-094739/P64_CANARY.md`
|
||||
- generated content: `P6.4 real Tool Gateway canary succeeded.`
|
||||
- Final state:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- all workers `heartbeatOk=true`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- real file writes limited to `/Users/john/Project/memind_canary/...`
|
||||
|
||||
### P6.5 OpenHands Real Canary
|
||||
|
||||
结果: 通过,OpenHands 真实执行 canary 已通过。
|
||||
|
||||
- OpenHands:
|
||||
- CLI: `1.16.0`
|
||||
- SDK: `1.21.0`
|
||||
- Launch:
|
||||
- command: `openhands`
|
||||
- args: `--headless --json --override-with-envs --task`
|
||||
- env from executor binding:
|
||||
- `LLM_MODEL=deepseek-reasoner`
|
||||
- `LLM_BASE_URL=https://api.deepseek.com/v1`
|
||||
- canary dir:
|
||||
- `/Users/john/Project/memind_canary/tool-gateway-p65-openhands-20260702-095402`
|
||||
- run id:
|
||||
- `0b1bb7a0-d111-44c1-91b5-4f14b2f9c01c`
|
||||
- request id:
|
||||
- `p65-real-openhands-20260702015402`
|
||||
- status:
|
||||
- `succeeded`
|
||||
- attempts: `1`
|
||||
- `agent_session_id=null`
|
||||
- duration:
|
||||
- about `206s`
|
||||
- event chain:
|
||||
- `queued`
|
||||
- `running`
|
||||
- `tool_gateway_dispatch`
|
||||
- `tool_gateway_result`
|
||||
- `succeeded`
|
||||
- `tool_gateway_dispatch.protocol=agent-run-v1`
|
||||
- `tool_gateway_dispatch.taskType=repo_refactor`
|
||||
- `tool_gateway_result.dryRun=false`
|
||||
- `tool_gateway_result.executor=openhands`
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- generated file:
|
||||
- `/Users/john/Project/memind_canary/tool-gateway-p65-openhands-20260702-095402/P65_OPENHANDS_CANARY.md`
|
||||
- generated content:
|
||||
- `P6.5 OpenHands Tool Gateway canary succeeded.`
|
||||
- Final state:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- all workers `heartbeatOk=true`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- real file writes limited to `/Users/john/Project/memind_canary/...`
|
||||
|
||||
### P6.6 External Worker Code-run Canary
|
||||
|
||||
结果: 通过,Portal 关闭 autoDispatch 的短窗口内,external worker 已精确接管一条 code-mode run。
|
||||
|
||||
- 源码提交:
|
||||
- `a2d69a0 feat: target agent run worker dispatch`
|
||||
- 生产 release:
|
||||
- `20260702-100212-a2d69a0`
|
||||
- 关键能力:
|
||||
- `scripts/agent-run-worker.mjs --run-id <id>` 可精确处理指定 run
|
||||
- canary 可通过 `MEMIND_AGENT_RUN_WORKDIR_OVERRIDE` 写入专用目录
|
||||
- Portal 短窗口:
|
||||
- `MEMIND_AGENT_RUN_AUTODISPATCH=0`
|
||||
- `MEMIND_AGENT_CODE_RUNS_ENABLED=1`
|
||||
- user allowlist 仅含 `a6fb1e97-2b0f-447b-b138-4561d8e5c53e`
|
||||
- 外部 worker:
|
||||
- `MEMIND_TOOL_GATEWAY_ENABLED=1`
|
||||
- `MEMIND_TOOL_GATEWAY_DRY_RUN=0`
|
||||
- workdir override: `/Users/john/Project/memind_canary/tool-gateway-p66-worker-20260702-100500`
|
||||
- run id:
|
||||
- `94dcca6f-cf32-4fa1-b602-8e6a5bff1e3c`
|
||||
- request id:
|
||||
- `p66-worker-aider-20260702100509`
|
||||
- worker output:
|
||||
- `ok=true`
|
||||
- `mode=dispatch`
|
||||
- `dispatched=1`
|
||||
- `runId=94dcca6f-cf32-4fa1-b602-8e6a5bff1e3c`
|
||||
- queue `autoDispatch=false`
|
||||
- Tool Gateway `enabled=true`
|
||||
- `dryRun=false`
|
||||
- event chain:
|
||||
- `queued`
|
||||
- `running`
|
||||
- `tool_gateway_dispatch`
|
||||
- `tool_gateway_result`
|
||||
- `succeeded`
|
||||
- `tool_gateway_result.dryRun=false`
|
||||
- `tool_gateway_result.executor=aider`
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- generated file:
|
||||
- `/Users/john/Project/memind_canary/tool-gateway-p66-worker-20260702-100500/P66_WORKER_CANARY.md`
|
||||
- generated content:
|
||||
- `P6.6 external worker Tool Gateway canary succeeded.`
|
||||
- Final state:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- all workers `heartbeatOk=true`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- real file writes limited to `/Users/john/Project/memind_canary/...`
|
||||
|
||||
### P6.7 Tool Gateway Production Guardrails
|
||||
|
||||
结果: 通过,产物校验/输出审计已发布,生产保持默认关闭。
|
||||
|
||||
- 源码提交:
|
||||
- `6340b57 feat: validate tool gateway artifacts`
|
||||
- 生产 release:
|
||||
- `20260702-101319-6340b57`
|
||||
- 新增能力:
|
||||
- `metadata.memindRun.validation.expectedFile`
|
||||
- `metadata.memindRun.validation.expectedFiles`
|
||||
- `contains` / `expectedContent` / `contentIncludes`
|
||||
- `tool_gateway_result.stdoutTail`
|
||||
- `tool_gateway_result.stderrTail`
|
||||
- `tool_gateway_validation`
|
||||
- `tool_gateway_validation_failed`
|
||||
- 失败语义:
|
||||
- executor exit 0 但 expected file/content 缺失时,run 标记 `failed`
|
||||
- validation failed 不重试
|
||||
- 校验路径必须位于 Tool Gateway working directory 内
|
||||
- 验证:
|
||||
- `node --test agent-run-gateway.test.mjs tool-gateway.test.mjs`: `17 passed`
|
||||
- `node --check agent-run-gateway.mjs`: passed
|
||||
- `npm run build:portal-runtime`: passed
|
||||
- 发布备注:
|
||||
- release 首次在已知历史 `duck-industry-report.docx` 缺失链接处停止
|
||||
- 按用户要求不修复历史用户数据,使用 `ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES=1` 仅跳过该守卫
|
||||
- release 脚本完成 live 全目录备份和持久目录备份
|
||||
- Final state:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
|
||||
### P6.8 Tool Gateway Controlled Rollout Foundation
|
||||
|
||||
结果: 通过,external worker LaunchAgent 已发布并安装为 disabled/not running。
|
||||
|
||||
- 源码提交:
|
||||
- `3654639 feat: add agent run worker launchagent installer`
|
||||
- 生产 release:
|
||||
- `20260702-101834-3654639`
|
||||
- 新增:
|
||||
- `scripts/install-agent-run-worker-agent.sh`
|
||||
- runtime artifact:
|
||||
- 已包含 `scripts/install-agent-run-worker-agent.sh`
|
||||
- RUNBOOK 已记录 `bash scripts/install-agent-run-worker-agent.sh`
|
||||
- LaunchAgent:
|
||||
- label: `cn.tkmind.memind-agent-run-worker`
|
||||
- plist: `/Users/john/Library/LaunchAgents/cn.tkmind.memind-agent-run-worker.plist`
|
||||
- log: `/Users/john/Library/Logs/memind-agent-run-worker.log`
|
||||
- 安装状态:
|
||||
- `state=installed-disabled`
|
||||
- `print-disabled`: disabled
|
||||
- `launchctl print`: `state = not running`
|
||||
- no `agent-run-worker.mjs` process
|
||||
- 默认配置:
|
||||
- `MEMIND_AGENT_RUN_AUTODISPATCH=0`
|
||||
- `MEMIND_AGENT_RUN_QUEUE_CONCURRENCY=1`
|
||||
- `MEMIND_TOOL_GATEWAY_ENABLED=0`
|
||||
- `MEMIND_TOOL_GATEWAY_DRY_RUN=0`
|
||||
- 验证:
|
||||
- `bash -n scripts/install-agent-run-worker-agent.sh`: passed
|
||||
- `node --check scripts/build-portal-runtime.mjs`: passed
|
||||
- `node --test agent-run-gateway.test.mjs tool-gateway.test.mjs`: `17 passed`
|
||||
- `npm run build:portal-runtime`: passed
|
||||
- Final state:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- external worker not started
|
||||
|
||||
### P6.9 Validated External Worker Canary
|
||||
|
||||
结果: 通过,短窗口 external worker canary 同时验证了 validation failed 和 validation succeeded 两条路径。
|
||||
|
||||
- 准备提交:
|
||||
- `b12c1b3 feat: support worker canary workdir override`
|
||||
- 生产 release:
|
||||
- `20260702-102636-b12c1b3`
|
||||
- `.env` backup:
|
||||
- `/Users/john/Project/memind_backups/20260702-1029-p69-validated-worker-canary/.env.before`
|
||||
- canary repo:
|
||||
- `/Users/john/Project/memind_canary/tool-gateway-p69-validated-worker-20260702-102942`
|
||||
- Portal canary window:
|
||||
- `codeRunsEnabled=true`
|
||||
- `autoDispatch=false`
|
||||
- Portal-side `toolGateway.enabled=false`
|
||||
- worker canary window:
|
||||
- LaunchAgent label: `cn.tkmind.memind-agent-run-worker`
|
||||
- `MEMIND_TOOL_GATEWAY_ENABLED=1`
|
||||
- `MEMIND_AGENT_RUN_WORKDIR_OVERRIDE=/Users/john/Project/memind_canary/tool-gateway-p69-validated-worker-20260702-102942`
|
||||
- `MEMIND_AGENT_RUN_QUEUE_CONCURRENCY=1`
|
||||
|
||||
Failure-path proof:
|
||||
|
||||
- run id:
|
||||
- `1353d917-c03b-4fb8-9acd-739d56984fc5`
|
||||
- request id:
|
||||
- `p69-validated-worker-20260702022958`
|
||||
- result:
|
||||
- `failed`
|
||||
- attempts: `1`
|
||||
- proof:
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- `tool_gateway_validation_failed.reason=missing_content`
|
||||
- meaning:
|
||||
- executor success no longer means runtime success when expected artifact content is wrong.
|
||||
|
||||
Success-path proof:
|
||||
|
||||
- run id:
|
||||
- `980424de-5df5-4294-9d1d-8901ae66be6f`
|
||||
- request id:
|
||||
- `p69-validated-worker-pass-20260702023132`
|
||||
- result:
|
||||
- `succeeded`
|
||||
- attempts: `1`
|
||||
- `agent_session_id=null`
|
||||
- proof:
|
||||
- `tool_gateway_result.exitCode=0`
|
||||
- `tool_gateway_validation.expectedFiles[0].path=P69_VALIDATED_WORKER_CANARY_PASS.md`
|
||||
- `tool_gateway_validation.expectedFiles[0].contains=true`
|
||||
- `tool_gateway_validation.expectedFiles[0].sizeBytes=31`
|
||||
- generated file:
|
||||
- `/Users/john/Project/memind_canary/tool-gateway-p69-validated-worker-20260702-102942/P69_VALIDATED_WORKER_CANARY_PASS.md`
|
||||
- generated content:
|
||||
- `P69_VALIDATED_WORKER_CANARY_OK`
|
||||
|
||||
Final state:
|
||||
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- worker LaunchAgent:
|
||||
- installed disabled
|
||||
- `state=not running`
|
||||
- no `agent-run-worker.mjs` process
|
||||
- temp login sessions:
|
||||
- `revokedRows=2`
|
||||
- SLO:
|
||||
- `ok=true`
|
||||
- `failures=[]`
|
||||
- writes all false
|
||||
|
||||
Data boundary:
|
||||
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- canary writes limited to `/Users/john/Project/memind_canary/...`
|
||||
|
||||
### P6.10 External Worker Observability
|
||||
|
||||
结果: 通过,external worker 只读状态检查已发布。
|
||||
|
||||
- 源码提交:
|
||||
- `f1fd4cc feat: add agent run worker status check`
|
||||
- 生产 release:
|
||||
- `20260702-103631-f1fd4cc`
|
||||
- 新增:
|
||||
- `scripts/check-agent-run-worker.mjs`
|
||||
- runtime artifact:
|
||||
- 已包含 `scripts/check-agent-run-worker.mjs`
|
||||
- RUNBOOK 已记录 `node scripts/check-agent-run-worker.mjs`
|
||||
- 只读检查覆盖:
|
||||
- LaunchAgent plist
|
||||
- launchctl state
|
||||
- disabled/enabled
|
||||
- pid/program/path
|
||||
- `agent-run-worker.mjs` process list
|
||||
- queued/running/retryable counts
|
||||
- oldest pending age
|
||||
- latest failed run
|
||||
- 验证:
|
||||
- `node --check scripts/check-agent-run-worker.mjs`: passed
|
||||
- `MEMIND_ENV_FILE=/Users/john/Project/Memind/.env node scripts/check-agent-run-worker.mjs`: `ok=true`
|
||||
- `node --test agent-run-gateway.test.mjs tool-gateway.test.mjs`: `17 passed`
|
||||
- `npm run build:portal-runtime`: passed
|
||||
- Final production check:
|
||||
- `check-agent-run-worker.ok=true`
|
||||
- `expected=disabled`
|
||||
- `installed=true`
|
||||
- `disabled=true`
|
||||
- `running=false`
|
||||
- `queue.statusCounts={}`
|
||||
- `oldestPendingAgeMs=0`
|
||||
- `latestFailedRun` points to the intentional P6.9 validation failure
|
||||
- Final runtime:
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- new script is read-only
|
||||
|
||||
### P6.11 Single-user Rollout Policy
|
||||
|
||||
结果: 通过,code-run 长期灰度所需策略门禁已发布,默认关闭。
|
||||
|
||||
- 源码提交:
|
||||
- `212e163 feat: add code run rollout policy gates`
|
||||
- 生产 release:
|
||||
- `20260702-104734-212e163`
|
||||
- 新增策略:
|
||||
- `MEMIND_AGENT_CODE_RUNS_ENABLED`
|
||||
- `MEMIND_AGENT_CODE_RUNS_USER_IDS`
|
||||
- `MEMIND_AGENT_CODE_RUN_TASK_TYPES`
|
||||
- `MEMIND_AGENT_CODE_RUNS_REQUIRE_VALIDATION`
|
||||
- 入口 gate:
|
||||
- 非用户 allowlist: `403 当前用户未开启代码任务灰度`
|
||||
- 非 task type allowlist: `403 当前代码任务类型未开启灰度`
|
||||
- 缺少 required validation: `400 代码任务必须声明产物校验规则`
|
||||
- runtime/status:
|
||||
- `toolRuntime.codeRunPolicy.enabled`
|
||||
- `toolRuntime.codeRunPolicy.userAllowlist`
|
||||
- `toolRuntime.codeRunPolicy.taskTypeAllowlist`
|
||||
- `toolRuntime.codeRunPolicy.requireValidation`
|
||||
- 验证:
|
||||
- `node --test agent-run-routes.test.mjs agent-run-gateway.test.mjs tool-gateway.test.mjs`: `31 passed`
|
||||
- `node --check agent-run-routes.mjs`: passed
|
||||
- `node --check server.mjs`: passed
|
||||
- `npm run build:portal-runtime`: passed
|
||||
- 发布备注:
|
||||
- release 脚本在 legacy `m.tkmind.cn` tunnel step 短暂返回失败。
|
||||
- Portal runtime 已切换,`mm.tkmind.cn` 主路径健康。
|
||||
- 后续检查显示 `cn.tkmind.memind-portal-tunnel` 已恢复 running。
|
||||
- 当前 H5 主路径仍为 `https://mm.tkmind.cn`,不依赖 105 转发。
|
||||
- Final runtime:
|
||||
- `toolRuntime.codeRunPolicy.enabled=false`
|
||||
- `toolRuntime.codeRunPolicy.userAllowlist=[]`
|
||||
- `toolRuntime.codeRunPolicy.taskTypeAllowlist=[]`
|
||||
- `toolRuntime.codeRunPolicy.requireValidation=false`
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.queue.autoDispatch=true`
|
||||
- `toolRuntime.queue.toolGateway.enabled=false`
|
||||
- Final checks:
|
||||
- `node scripts/check-agent-run-worker.mjs`: `ok=true`, worker disabled/not running
|
||||
- SLO `ok=true`, `failures=[]`, writes all false
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
- no real-user code run enabled
|
||||
|
||||
### P6.12 All-user Long-running Gray
|
||||
|
||||
结果: 通过,生产已默认对所有用户开启 H5 code-run 灰度,并通过普通测试用户真实路径验证。
|
||||
|
||||
- 生产变更:
|
||||
- `MEMIND_AGENT_CODE_RUNS_ENABLED=1`
|
||||
- `MEMIND_AGENT_CODE_RUNS_USER_IDS=`,空值表示所有用户。
|
||||
- `MEMIND_AGENT_CODE_RUN_TASK_TYPES=`,空值表示所有任务类型。
|
||||
- `MEMIND_AGENT_CODE_RUNS_REQUIRE_VALIDATION=0`,兼容 H5 默认 code-run payload;带 validation 的任务仍会校验。
|
||||
- `MEMIND_AGENT_RUN_AUTODISPATCH=0`
|
||||
- external worker `cn.tkmind.memind-agent-run-worker` enabled/running。
|
||||
- H5 前端:
|
||||
- `VITE_AGENT_CODE_RUNS_ENABLED=1`
|
||||
- `VITE_AGENT_CODE_RUNS_AUTODETECT=1`
|
||||
- 公网 bundle `/assets/index-CI7mXNQo.js`
|
||||
- 备份:
|
||||
- `/Users/john/Project/memind_backups/20260702-111654-all-user-code-run-gray`
|
||||
- 普通用户验证:
|
||||
- user `gray20260702112309`
|
||||
- user id `b4561354-4273-4900-9ada-b6c52d2c5836`
|
||||
- run id `aa0d1ac1-c03b-4110-baea-6cac83556567`
|
||||
- request id `gray-code-20260702112309-login`
|
||||
- status `succeeded`
|
||||
- attempts `1`
|
||||
- 事件链:
|
||||
- `queued`
|
||||
- `running`
|
||||
- `tool_gateway_dispatch`
|
||||
- `tool_gateway_result`
|
||||
- `succeeded`
|
||||
- Tool Gateway:
|
||||
- executor `aider`
|
||||
- `dryRun=false`
|
||||
- working dir `/Users/john/Project/Memind/MindSpace/b4561354-4273-4900-9ada-b6c52d2c5836`
|
||||
- 产物:
|
||||
- `/Users/john/Project/Memind/MindSpace/b4561354-4273-4900-9ada-b6c52d2c5836/public/gray-code-run-20260702112309.html`
|
||||
- content `GRAY_CODE_RUN_20260702112309`
|
||||
- 观察:
|
||||
- `check-agent-run-worker`: `ok=true`, worker running, queue empty
|
||||
- SLO: `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no existing production MindSpace deletion or rewrite
|
||||
- real writes limited to a new test user's MindSpace
|
||||
|
||||
### P6.13 Agent Run Auto-pause Guard
|
||||
|
||||
结果: 通过,自动暂停 guard 已发布到生产 runtime 并安装为 LaunchAgent。
|
||||
|
||||
- 新增源码:
|
||||
- `scripts/agent-run-guard.mjs`
|
||||
- `scripts/install-agent-run-guard-agent.sh`
|
||||
- `docs/agent-run-worker-rollout-runbook.md`
|
||||
- `scripts/build-portal-runtime.mjs` runtime copy list updated
|
||||
- 生产安装:
|
||||
- label `cn.tkmind.memind-agent-run-guard`
|
||||
- interval `60s`
|
||||
- log `/Users/john/Library/Logs/memind-agent-run-guard.log`
|
||||
- 默认阈值:
|
||||
- failed runs in 10 minutes `>= 3`
|
||||
- oldest queued/retryable age `>= 300000ms`
|
||||
- queued + retryable count `>= 10`
|
||||
- oldest running age `>= 900000ms`
|
||||
- 触发动作:
|
||||
- backup `.env`
|
||||
- set `MEMIND_AGENT_CODE_RUNS_ENABLED=0`
|
||||
- set `MEMIND_AGENT_RUN_AUTODISPATCH=1`
|
||||
- stop and disable `cn.tkmind.memind-agent-run-worker`
|
||||
- kickstart `cn.tkmind.memind-portal`
|
||||
- 备份:
|
||||
- `/Users/john/Project/memind_backups/20260702-113106-agent-run-guard-install`
|
||||
- 验证:
|
||||
- `node scripts/agent-run-guard.mjs --dry-run`: `ok=true`, `shouldPause=false`
|
||||
- guard LaunchAgent enabled, last exit code `0`
|
||||
- worker remains running
|
||||
- all-user code-run gate remains enabled
|
||||
- SLO `ok=true`, `failures=[]`
|
||||
- Data boundary:
|
||||
- no schema migration
|
||||
- no PG data deletion
|
||||
- no production MindSpace file deletion or rewrite
|
||||
|
||||
### 入口与 Gateway
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- `https://mm.tkmind.cn/api/status` 返回 `ok`。
|
||||
- `H5_PUBLIC_BASE_URL=https://mm.tkmind.cn`。
|
||||
- `m.tkmind.cn` 已作为 legacy/rollback 记录,不作为后续 H5 主路径。
|
||||
- nginx `mm.tkmind.cn` 已对 SSE 路由禁用 buffering/cache/gzip,并设置长 read/send timeout。
|
||||
- SSE route:
|
||||
- `/api/sessions/<id>/events`
|
||||
- `/api/agent/runs/<id>/events`
|
||||
- `x-accel-buffering: no` 检查通过。
|
||||
|
||||
评价:
|
||||
|
||||
- Gateway 层已符合 stream-first 的基本要求。
|
||||
- 非 SSE 普通 route 仍走统一 `location /`,当前足够。
|
||||
- 后续可加 nginx 级别 request id 和 upstream timing log,便于首 token 延迟统计。
|
||||
|
||||
### Stream Controller
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- `scripts/check-stream-runtime.mjs` 返回 `ok=true`。
|
||||
- SSE HEAD 探测通过,不会打开真实上游 stream。
|
||||
- Portal 对 session events 和 agent run events 均具备 no-buffering 响应头。
|
||||
- Session SSE 已记录 worker 级 first-token latency EWMA。
|
||||
- Session SSE 已记录最近 5 分钟/1 小时 first-token p50/p95。
|
||||
|
||||
评价:
|
||||
|
||||
- 已完成 P0/P3.5 级别能力。
|
||||
- 当前已有 first-token EWMA、p50/p95 窗口趋势和 stream lifecycle counters。
|
||||
- 后续建议增加日报落盘或更长周期趋势,不进 PG。
|
||||
|
||||
### Redis Router / Control Plane
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- Redis 容器 `memind-runtime-redis` 正常运行。
|
||||
- `MEMIND_RUNTIME_REDIS_URL=redis://127.0.0.1:6379/0`。
|
||||
- namespace: `memind:runtime`。
|
||||
- `/api/runtime/status.router.enabled=true`。
|
||||
- worker 状态可见:
|
||||
- `activeStreams`
|
||||
- `streamOpenCount`
|
||||
- `streamAbortCount`
|
||||
- `streamErrorCount`
|
||||
- `score`
|
||||
- `drain`
|
||||
|
||||
评价:
|
||||
|
||||
- Control Plane 已实际承担 runtime state。
|
||||
- P5.8 已安装 LaunchAgent,每 60 秒写入 worker metrics,Router 不再只靠请求流量刷新 worker 状态。
|
||||
- 当前仍不是 worker self-heartbeat,后续可以改为 sidecar 或 worker 内部心跳。
|
||||
|
||||
### Goose Worker Pool
|
||||
|
||||
结果: 通过。
|
||||
|
||||
四个 target 均 healthy:
|
||||
|
||||
- `https://127.0.0.1:18006`
|
||||
- `https://127.0.0.1:18007`
|
||||
- `https://127.0.0.1:18008`
|
||||
- `https://127.0.0.1:18009`
|
||||
|
||||
评价:
|
||||
|
||||
- Worker pool 已可水平调度,并具备 drain 运维入口。
|
||||
- 当前调度 scoring 已接入 CPU / memory / fd pressure 的采样指标。
|
||||
- first-token latency 已接入 Redis EWMA,并参与 Router score。
|
||||
- first-token p50/p95 已按 5 分钟和 1 小时窗口聚合。
|
||||
|
||||
### Runtime Metrics / SLO
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- 已安装 LaunchAgent `cn.tkmind.memind-runtime-metrics`。
|
||||
- 默认每 60 秒执行 `scripts/runtime-worker-metrics.mjs sample`。
|
||||
- 已安装 LaunchAgent `cn.tkmind.memind-runtime-heartbeat`。
|
||||
- 默认每 15 秒执行 worker `/status` heartbeat sidecar。
|
||||
- `launchctl print` 显示最近运行退出码为 `0`。
|
||||
- `scripts/runtime-slo-report.mjs` 返回:
|
||||
- `ok=true`
|
||||
- `failures=[]`
|
||||
- 四个 worker `metricsFresh=true`
|
||||
- 已展示 `ewmaFirstTokenMs`、`lastFirstTokenMs`、`firstTokenCount`
|
||||
- 已展示 `firstToken5m`、`firstToken1h`
|
||||
- 已展示 `toolQueue`
|
||||
- 已展示 `heartbeatSource=sidecar`
|
||||
- 已展示 `heartbeatFresh=true`
|
||||
- 已展示 `heartbeatOk=true`
|
||||
- `writes.database=false`
|
||||
- `writes.mindSpace=false`
|
||||
- `writes.redis=false`
|
||||
- 生产已观测到真实样本:
|
||||
- `goosed-4.ewmaFirstTokenMs=125`
|
||||
- `goosed-4.firstTokenCount=1`
|
||||
- `goosed-4.firstToken5m.p50Ms=188`
|
||||
- `goosed-4.firstToken5m.p95Ms=188`
|
||||
|
||||
评价:
|
||||
|
||||
- Observability 从人工检查推进到定时采样 + 一键 SLO 快照。
|
||||
- SLO 报告会读取 PG 表行数和 MindSpace 文件统计,但不写入、不删除用户数据。
|
||||
- 已有 Redis rolling samples、即时窗口趋势和每日自动 JSON/Markdown SLO 日报。
|
||||
- P5.13 已给日报目录增加 30 天保留策略:
|
||||
- LaunchAgent 参数包含 `--prune --retention-days 30`
|
||||
- 只清理 `reports/runtime-slo` 顶层过期 `.json/.md`
|
||||
- 最新报告 `reportPrune.errors=[]`
|
||||
- P5.14 已给四个 goosed worker 增加常驻 sidecar heartbeat:
|
||||
- `goosed-1`: `statusCode=200`, `latencyMs=62`
|
||||
- `goosed-2`: `statusCode=200`, `latencyMs=46`
|
||||
- `goosed-3`: `statusCode=200`, `latencyMs=46`
|
||||
- `goosed-4`: `statusCode=200`, `latencyMs=53`
|
||||
|
||||
### Tool Runtime / Aider / OpenHands
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- `scripts/check-tool-runtime.mjs` 返回 `ok=true`。
|
||||
- `role=user` 默认:
|
||||
- `aider=false`
|
||||
- `openhands=false`
|
||||
- 显式白名单用户数: 2。
|
||||
- 白名单用户:
|
||||
- chat mode 不注入 Aider/OpenHands。
|
||||
- code mode 可注入 Aider/OpenHands。
|
||||
- `toolRuntime.chatInjectsCodeTools=false`。
|
||||
- `toolRuntime.codeRunsEnabled=false`。
|
||||
- `toolRuntime.queue` 已上线:
|
||||
- `autoDispatch=true`
|
||||
- `maxConcurrentRuns=1`
|
||||
- `runTimeoutMs=900000`
|
||||
- `inFlight=0`
|
||||
- `pendingDispatches=0`
|
||||
- `scripts/agent-run-worker.mjs --status` 已可在生产 runtime 中独立运行。
|
||||
- `scripts/agent-run-worker.mjs --once` 已完成真实 chat-mode canary:
|
||||
- `8e64ff4b-c043-475e-b60b-ae7466111da9` -> `succeeded`
|
||||
|
||||
评价:
|
||||
|
||||
- 已完成高风险工具从普通聊天默认能力中的隔离。
|
||||
- 后端 `MEMIND_AGENT_CODE_RUNS_ENABLED` 默认关闭,防止客户端绕过前端灰度。
|
||||
- `/agent/runs` 已具备最小 queue gateway: 全局并发限制、运行超时、runtime/SLO 可观测状态。
|
||||
- 外部 worker 接管入口已具备,并通过短窗口真实 canary 验证。
|
||||
- 生产已恢复 Portal 内部 `autoDispatch=true`。
|
||||
- Aider/OpenHands 仍是 session extension 形态,尚未拆成独立工具进程协议。
|
||||
|
||||
### H5 Code Run 入口
|
||||
|
||||
结果: 后端 canary 与 H5 UI canary 均已通过,默认关闭。
|
||||
|
||||
- H5 bundle 已包含 `/agent/runs` 和 `tool_mode` 入口。
|
||||
- `VITE_AGENT_CODE_RUNS_ENABLED` 未设置。
|
||||
- `VITE_AGENT_CODE_RUNS_AUTODETECT` 未设置。
|
||||
- `VITE_AGENT_CODE_RUNS_USER_IDS` 未设置。
|
||||
- `MEMIND_AGENT_CODE_RUNS_ENABLED` 未设置。
|
||||
- `MEMIND_AGENT_CODE_RUNS_USER_IDS` 未设置。
|
||||
- `scripts/check-agent-code-run-entry.mjs` 返回 `ok=true`。
|
||||
- P4.10 后端短窗口 code-mode canary 已成功:
|
||||
- run id: `805e7422-175e-4eb7-8fb2-df0c29f79c0c`
|
||||
- session id: `20260702_3`
|
||||
- final status: `succeeded`
|
||||
- P4.11 已发布默认关闭 runtime:
|
||||
- release id: `20260702-085001-a0af049`
|
||||
- `toolRuntime.codeRunsEnabled=false`
|
||||
- `toolRuntime.chatInjectsCodeTools=false`
|
||||
- 发布脚本已支持把 `ALLOW_MINDSPACE_PUBLIC_LINK_ISSUES=1` 传到远端 post-check。
|
||||
- 已按用户要求忽略既有生产 MindSpace 公开页缺失 docx 附件链接,不修复。
|
||||
- P4.12 H5 页面编辑 UI canary 已完成:
|
||||
- canary release: `20260702-085319-a0af049`
|
||||
- canary run id: `825720c9-330d-447d-a6ed-afeabbf6e14c`
|
||||
- canary session id: `20260702_4`
|
||||
- final status: `succeeded`
|
||||
- attempts: `1`
|
||||
- restored release: `20260702-085706-a0af049`
|
||||
- restored `toolRuntime.codeRunsEnabled=false`
|
||||
- restored `.env` 无 `MEMIND_AGENT_CODE_RUNS_*`
|
||||
- 脚本明确:
|
||||
- `writes.database=false`
|
||||
- `writes.mindSpace=false`
|
||||
- `createsAgentRun=false`
|
||||
|
||||
评价:
|
||||
|
||||
- 灰度链路已经有前后端双门禁,并补齐用户级白名单。
|
||||
- 当前生产不会自动启用 code run。
|
||||
- 当前已恢复默认关闭;后续如要放量,可只对白名单重新打开,不开普通聊天 autodetect。
|
||||
|
||||
### Persistence / PG / MindSpace
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- 本轮 P4.6-P6.2/P4.10 未做 schema migration。
|
||||
- 本轮未删除 PG 数据。
|
||||
- 本轮未删除 MindSpace 数据。
|
||||
- P5.8 SLO 报告只读统计 PG 和 MindSpace。
|
||||
- P5.13 SLO 报告保留策略只清理运维报告目录。
|
||||
- P6.2/P4.10 仅新增 canary run/session/event 审计记录。
|
||||
- 当前 `MindSpace`:
|
||||
- `/Users/john/Project/Memind/MindSpace`: `files=2143`, `dirs=1595`, `bytes=122160654`
|
||||
- `/Users/john/Project/Memind/data/mindspace`: `files=5281`, `dirs=7651`, `bytes=270214686`
|
||||
- 已有生产备份:
|
||||
- `20260702-065813-pre-p35-p55`
|
||||
- `20260702-071604-pre-p46-code-task-entry`
|
||||
- `20260702-071914-pre-p47-h5-code-run-entry`
|
||||
- `20260702-072233-pre-p48-code-run-check`
|
||||
- `20260702-072452-pre-p49-code-run-server-gate`
|
||||
- `20260702-073927-pre-p58-runtime-slo-agent`
|
||||
- `20260702-074955-pre-p60-tool-queue-v0`
|
||||
- `20260702-075904-pre-p61-agent-run-worker-v1`
|
||||
- `20260702-081928-pre-p62-tool-worker-canary`
|
||||
- `20260702-083225-pre-p410-code-run-canary`
|
||||
- `20260702-083917-pre-p513-slo-prune`
|
||||
- `memind-full-20260702-084418-c7d8140-before.tar.gz`
|
||||
- `memind-persisted-20260702-084418-c7d8140-before.tar.gz`
|
||||
- `memind-full-20260702-085001-a0af049-before.tar.gz`
|
||||
- `memind-persisted-20260702-085001-a0af049-before.tar.gz`
|
||||
- `20260702-085302-pre-p412-h5-ui-canary`
|
||||
- `memind-full-20260702-085319-a0af049-before.tar.gz`
|
||||
- `memind-persisted-20260702-085319-a0af049-before.tar.gz`
|
||||
- `memind-full-20260702-085706-a0af049-before.tar.gz`
|
||||
- `memind-persisted-20260702-085706-a0af049-before.tar.gz`
|
||||
- `memind-full-20260702-090343-e24e6ca-before.tar.gz`
|
||||
- `memind-persisted-20260702-090343-e24e6ca-before.tar.gz`
|
||||
|
||||
评价:
|
||||
|
||||
- 数据保护执行良好。
|
||||
- 建议每次进入“会创建真实 agent run 的 canary”前,再做一次轻量 DB/MindSpace 快照或至少记录 row/file count baseline。
|
||||
|
||||
### Source Sync / Remote Dev
|
||||
|
||||
结果: 通过。
|
||||
|
||||
- 干净分支当前 clean。
|
||||
- 分支已推送:
|
||||
- `memind-streaming-runtime-20260702`
|
||||
- 远程开发机同步命令:
|
||||
- `git pull origin memind-streaming-runtime-20260702`
|
||||
|
||||
评价:
|
||||
|
||||
- 已满足“远程开发电脑保持一致、后续直接拉分支”的要求。
|
||||
- 生产 runtime 是单文件 artifact,源码分支保留 `.runtime/portal/server.mjs` 便于复现生产部署。
|
||||
|
||||
## 当前架构完成度
|
||||
|
||||
| 模块 | 完成度 | 状态 |
|
||||
| --- | ---: | --- |
|
||||
| Edge Gateway / mm domain | 85% | 已切主路径,SSE 配置正确 |
|
||||
| Stream Controller | 86% | pipeline 已落地,first-token EWMA 和 p50/p95 已接入 |
|
||||
| Redis Control Plane | 82% | runtime state 已迁移,worker sidecar heartbeat 已上线 |
|
||||
| Worker Pool | 86% | 四 worker healthy,drain 可用,CPU/memory/fd 指标和主动 heartbeat 已接入 |
|
||||
| Tool Isolation | 99% | chat/code 隔离完成,queue gateway v0、外部 worker 入口、真实 canary、`agent-run-v1` 协议、Aider/OpenHands 真实 canary、产物校验、worker LaunchAgent、validated worker canary、worker check、rollout gates、全用户灰度和自动暂停 guard 已完成 |
|
||||
| H5 Code Run Canary | 99% | 后端 code-mode canary、用户级 gate、H5 页面编辑 UI canary、external worker 精确接管 canary、guardrails、validated worker canary、rollout policy、全用户真实路径验证已完成,当前生产处于 all-user gray |
|
||||
| Persistence Boundary | 85% | PG/MindSpace 保护良好,SLO 脚本只读验证 |
|
||||
| Observability | 96% | 定时 metrics、sidecar heartbeat、SLO 快照/日报/30天保留、first-token EWMA 与 p50/p95 已落地 |
|
||||
| Remote Sync | 90% | 干净分支已稳定推送 |
|
||||
|
||||
## 风险清单
|
||||
|
||||
1. activeStreams 可能残留。
|
||||
- 影响: score 会偏高,长期可能影响调度。
|
||||
- 状态: P5.15 Active Stream TTL Reconcile 已按用户要求跳过,暂不做报表/定时 reconcile。
|
||||
|
||||
2. Tool Gateway 已进入全用户长期灰度,需持续观察。
|
||||
- 影响: code-run 现在会对所有 H5 用户自动启用;external worker 单并发处理,guard 会在失败或积压超过阈值时自动暂停。
|
||||
- 状态: P6.12/P6.13 已完成;当前 `codeRunsEnabled=true`、`autoDispatch=false`、external worker running、guard enabled。
|
||||
- 建议: 保持 30-60 分钟密集观察,然后转为日报/SLO + guard 常态观察。
|
||||
|
||||
3. 生产 MindSpace 存在既有公开页缺失 docx 附件链接。
|
||||
- 影响: 已按用户要求作为历史内容问题忽略,不阻断 streaming runtime 发布。
|
||||
- 建议: 不在本轮修复;后续只有用户重新要求时再单独处理。
|
||||
|
||||
## 下一步执行建议
|
||||
|
||||
### P6.14 Gray Ops Observation
|
||||
|
||||
- 观察 30-60 分钟:
|
||||
- `MEMIND_AGENT_RUN_WORKER_EXPECT_RUNNING=1 node scripts/check-agent-run-worker.mjs`
|
||||
- `node scripts/agent-run-guard.mjs --dry-run`
|
||||
- `node scripts/runtime-slo-report.mjs`
|
||||
- `tail -200 ~/Library/Logs/memind-agent-run-worker.log`
|
||||
- `tail -200 ~/Library/Logs/memind-agent-run-guard.log`
|
||||
- 重点判断:
|
||||
- queue 是否持续为空或快速归零。
|
||||
- 是否出现新的 failed run。
|
||||
- guard 是否误触发或正确保持 `shouldPause=false`。
|
||||
- SLO 是否仍 `failures=[]`。
|
||||
- 观察稳定后再评估:
|
||||
- 是否保持 worker 单并发。
|
||||
- 是否把 validation metadata 自动补入 H5 默认 code-run。
|
||||
- 是否增加任务级产物规范和用户可见进度/失败说明。
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+305
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import mysql from 'mysql2/promise';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const DEFAULT_WORKER_LABEL = 'cn.tkmind.memind-agent-run-worker';
|
||||
const DEFAULT_PORTAL_LABEL = 'cn.tkmind.memind-portal';
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
let value = trimmed.slice(idx + 1).trim();
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function truthy(value) {
|
||||
return ['1', 'true', 'yes', 'on'].includes(String(value ?? '').trim().toLowerCase());
|
||||
}
|
||||
|
||||
function positiveInt(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
apply: argv.includes('--apply'),
|
||||
dryRun: argv.includes('--dry-run') || !argv.includes('--apply'),
|
||||
help: argv.includes('--help') || argv.includes('-h'),
|
||||
};
|
||||
return args;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log([
|
||||
'Usage:',
|
||||
' node scripts/agent-run-guard.mjs [--dry-run]',
|
||||
' node scripts/agent-run-guard.mjs --apply',
|
||||
'',
|
||||
'Dry-run is the default. --apply can stop the external worker and disable code-run gate in .env.',
|
||||
].join('\n'));
|
||||
}
|
||||
|
||||
function parseMysqlConfig() {
|
||||
if (process.env.DATABASE_URL) {
|
||||
const url = new URL(process.env.DATABASE_URL);
|
||||
if (url.protocol !== 'mysql:') {
|
||||
throw new Error(`Unsupported DATABASE_URL scheme for agent run guard: ${url.protocol}`);
|
||||
}
|
||||
return {
|
||||
host: url.hostname,
|
||||
port: Number(url.port || 3306),
|
||||
user: decodeURIComponent(url.username),
|
||||
password: decodeURIComponent(url.password),
|
||||
database: url.pathname.replace(/^\/+/, ''),
|
||||
charset: 'utf8mb4',
|
||||
};
|
||||
}
|
||||
return {
|
||||
host: process.env.MYSQL_HOST,
|
||||
port: Number(process.env.MYSQL_PORT || 3306),
|
||||
user: process.env.MYSQL_USER,
|
||||
password: process.env.MYSQL_PASSWORD,
|
||||
database: process.env.MYSQL_DATABASE,
|
||||
charset: 'utf8mb4',
|
||||
};
|
||||
}
|
||||
|
||||
async function runCommand(command, args) {
|
||||
try {
|
||||
const { stdout, stderr } = await execFileAsync(command, args, { maxBuffer: 1024 * 1024 });
|
||||
return { ok: true, stdout, stderr };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
code: err?.code ?? null,
|
||||
stdout: err?.stdout ?? '',
|
||||
stderr: err?.stderr ?? '',
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function replaceOrAppendEnv(raw, updates) {
|
||||
const pending = new Map(Object.entries(updates));
|
||||
const lines = raw.split('\n');
|
||||
const next = lines.map((line) => {
|
||||
const match = line.match(/^(\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*=).*/);
|
||||
if (!match) return line;
|
||||
const key = match[2];
|
||||
if (!pending.has(key)) return line;
|
||||
const value = pending.get(key);
|
||||
pending.delete(key);
|
||||
return `${key}=${value}`;
|
||||
});
|
||||
if (pending.size > 0) {
|
||||
if (next.length > 0 && next[next.length - 1] !== '') next.push('');
|
||||
next.push('## agent-run guard auto-disable');
|
||||
for (const [key, value] of pending) next.push(`${key}=${value}`);
|
||||
}
|
||||
return next.join('\n').replace(/\n*$/, '\n');
|
||||
}
|
||||
|
||||
function timestamp() {
|
||||
const d = new Date();
|
||||
const pad = (n) => String(n).padStart(2, '0');
|
||||
return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}-${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
|
||||
}
|
||||
|
||||
async function readQueueHealth(now) {
|
||||
const conn = await mysql.createConnection(parseMysqlConfig());
|
||||
try {
|
||||
const [statusRows] = await conn.query(
|
||||
`SELECT status, COUNT(*) AS count
|
||||
FROM h5_agent_runs
|
||||
WHERE status IN ('queued', 'running', 'retryable')
|
||||
GROUP BY status`,
|
||||
);
|
||||
const statusCounts = {};
|
||||
for (const row of statusRows) statusCounts[row.status] = Number(row.count ?? 0);
|
||||
|
||||
const [oldestPendingRows] = await conn.query(
|
||||
`SELECT MIN(updated_at) AS oldest_updated_at
|
||||
FROM h5_agent_runs
|
||||
WHERE status IN ('queued', 'retryable')`,
|
||||
);
|
||||
const oldestPendingUpdatedAt = oldestPendingRows[0]?.oldest_updated_at == null
|
||||
? null
|
||||
: Number(oldestPendingRows[0].oldest_updated_at);
|
||||
|
||||
const [oldestRunningRows] = await conn.query(
|
||||
`SELECT MIN(started_at) AS oldest_started_at
|
||||
FROM h5_agent_runs
|
||||
WHERE status = 'running'`,
|
||||
);
|
||||
const oldestRunningStartedAt = oldestRunningRows[0]?.oldest_started_at == null
|
||||
? null
|
||||
: Number(oldestRunningRows[0].oldest_started_at);
|
||||
|
||||
const failedWindowMs = positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_FAILED_WINDOW_MS, 10 * 60 * 1000);
|
||||
const since = now - failedWindowMs;
|
||||
const [failedRows] = await conn.query(
|
||||
`SELECT COUNT(*) AS count
|
||||
FROM h5_agent_runs
|
||||
WHERE status = 'failed' AND updated_at >= ?`,
|
||||
[since],
|
||||
);
|
||||
const [latestFailedRows] = await conn.query(
|
||||
`SELECT id, request_id, error_message, updated_at
|
||||
FROM h5_agent_runs
|
||||
WHERE status = 'failed'
|
||||
ORDER BY updated_at DESC
|
||||
LIMIT 1`,
|
||||
);
|
||||
|
||||
return {
|
||||
statusCounts,
|
||||
queuedOrRetryable: Number(statusCounts.queued ?? 0) + Number(statusCounts.retryable ?? 0),
|
||||
running: Number(statusCounts.running ?? 0),
|
||||
oldestPendingUpdatedAt,
|
||||
oldestPendingAgeMs: oldestPendingUpdatedAt == null ? 0 : Math.max(0, now - oldestPendingUpdatedAt),
|
||||
oldestRunningStartedAt,
|
||||
oldestRunningAgeMs: oldestRunningStartedAt == null ? 0 : Math.max(0, now - oldestRunningStartedAt),
|
||||
failedWindowMs,
|
||||
failedRecentCount: Number(failedRows[0]?.count ?? 0),
|
||||
latestFailedRun: latestFailedRows[0] ? {
|
||||
id: latestFailedRows[0].id,
|
||||
requestId: latestFailedRows[0].request_id,
|
||||
error: latestFailedRows[0].error_message,
|
||||
updatedAt: Number(latestFailedRows[0].updated_at ?? 0),
|
||||
} : null,
|
||||
};
|
||||
} finally {
|
||||
await conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
function evaluateHealth(queue) {
|
||||
const thresholds = {
|
||||
maxRecentFailures: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES, 3),
|
||||
maxPendingAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS, 5 * 60 * 1000),
|
||||
maxPendingCount: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT, 10),
|
||||
maxRunningAgeMs: positiveInt(process.env.MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS, 15 * 60 * 1000),
|
||||
};
|
||||
const reasons = [];
|
||||
if (queue.failedRecentCount >= thresholds.maxRecentFailures) {
|
||||
reasons.push(`recent_failed_count ${queue.failedRecentCount} >= ${thresholds.maxRecentFailures}`);
|
||||
}
|
||||
if (queue.oldestPendingAgeMs >= thresholds.maxPendingAgeMs) {
|
||||
reasons.push(`oldest_pending_age_ms ${queue.oldestPendingAgeMs} >= ${thresholds.maxPendingAgeMs}`);
|
||||
}
|
||||
if (queue.queuedOrRetryable >= thresholds.maxPendingCount) {
|
||||
reasons.push(`pending_count ${queue.queuedOrRetryable} >= ${thresholds.maxPendingCount}`);
|
||||
}
|
||||
if (queue.oldestRunningAgeMs >= thresholds.maxRunningAgeMs) {
|
||||
reasons.push(`oldest_running_age_ms ${queue.oldestRunningAgeMs} >= ${thresholds.maxRunningAgeMs}`);
|
||||
}
|
||||
return { thresholds, reasons, shouldPause: reasons.length > 0 };
|
||||
}
|
||||
|
||||
async function disableCodeRuns({ root, envFile, workerLabel, portalLabel, gui, reasons, dryRun }) {
|
||||
const actions = [];
|
||||
const updates = {
|
||||
MEMIND_AGENT_CODE_RUNS_ENABLED: '0',
|
||||
MEMIND_AGENT_RUN_AUTODISPATCH: '1',
|
||||
};
|
||||
if (dryRun) {
|
||||
return {
|
||||
applied: false,
|
||||
actions: [
|
||||
`would backup ${envFile}`,
|
||||
`would set ${Object.entries(updates).map(([k, v]) => `${k}=${v}`).join(', ')}`,
|
||||
`would stop and disable ${workerLabel}`,
|
||||
`would kickstart ${portalLabel}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const backupRoot = process.env.MEMIND_AGENT_RUN_GUARD_BACKUP_DIR
|
||||
|| path.join(path.dirname(root), 'memind_backups', `${timestamp()}-agent-run-guard-pause`);
|
||||
fs.mkdirSync(backupRoot, { recursive: true });
|
||||
const envBackup = path.join(backupRoot, '.env.before');
|
||||
fs.copyFileSync(envFile, envBackup);
|
||||
actions.push(`backed_up_env:${envBackup}`);
|
||||
|
||||
const raw = fs.readFileSync(envFile, 'utf8');
|
||||
const updated = replaceOrAppendEnv(raw, updates);
|
||||
const marker = [
|
||||
'',
|
||||
`# agent-run guard pause at ${new Date().toISOString()}`,
|
||||
`# reasons: ${reasons.join('; ')}`,
|
||||
].join('\n');
|
||||
fs.writeFileSync(envFile, `${updated.replace(/\n*$/, '\n')}${marker}\n`, 'utf8');
|
||||
actions.push(`updated_env:${envFile}`);
|
||||
|
||||
await runCommand('launchctl', ['bootout', `${gui}/${workerLabel}`]);
|
||||
await runCommand('launchctl', ['disable', `${gui}/${workerLabel}`]);
|
||||
actions.push(`disabled_worker:${workerLabel}`);
|
||||
|
||||
const kick = await runCommand('launchctl', ['kickstart', '-k', `${gui}/${portalLabel}`]);
|
||||
actions.push(kick.ok ? `kickstarted_portal:${portalLabel}` : `portal_kickstart_failed:${kick.message}`);
|
||||
return { applied: true, backupRoot, actions };
|
||||
}
|
||||
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const root = path.join(path.dirname(new URL(import.meta.url).pathname), '..');
|
||||
const envFile = process.env.MEMIND_ENV_FILE || path.join(root, '.env');
|
||||
loadEnvFile(envFile);
|
||||
|
||||
const enabled = process.env.MEMIND_AGENT_RUN_GUARD_ENABLED == null
|
||||
? true
|
||||
: truthy(process.env.MEMIND_AGENT_RUN_GUARD_ENABLED);
|
||||
const workerLabel = process.env.MEMIND_AGENT_RUN_WORKER_LABEL || DEFAULT_WORKER_LABEL;
|
||||
const portalLabel = process.env.MEMIND_PORTAL_LABEL || DEFAULT_PORTAL_LABEL;
|
||||
const gui = `gui/${process.getuid()}`;
|
||||
const now = Date.now();
|
||||
const queue = await readQueueHealth(now).catch((err) => ({
|
||||
error: err instanceof Error ? (err.message || err.code || err.name) : String(err),
|
||||
}));
|
||||
const evaluation = queue.error
|
||||
? { thresholds: {}, reasons: [`queue_health_error ${queue.error}`], shouldPause: false }
|
||||
: evaluateHealth(queue);
|
||||
let pause = { applied: false, actions: [] };
|
||||
|
||||
if (enabled && evaluation.shouldPause) {
|
||||
pause = await disableCodeRuns({
|
||||
root,
|
||||
envFile,
|
||||
workerLabel,
|
||||
portalLabel,
|
||||
gui,
|
||||
reasons: evaluation.reasons,
|
||||
dryRun: args.dryRun,
|
||||
});
|
||||
}
|
||||
|
||||
const result = {
|
||||
ok: enabled ? !evaluation.shouldPause || pause.applied || args.dryRun : true,
|
||||
checkedAt: new Date(now).toISOString(),
|
||||
enabled,
|
||||
mode: args.apply ? 'apply' : 'dry-run',
|
||||
queue,
|
||||
evaluation,
|
||||
pause,
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(result.ok ? 0 : 1);
|
||||
@@ -335,6 +335,14 @@ async function writeMetadata() {
|
||||
path.join(root, 'scripts', 'check-agent-run-worker.mjs'),
|
||||
path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'),
|
||||
);
|
||||
await fs.copyFile(
|
||||
path.join(root, 'scripts', 'agent-run-guard.mjs'),
|
||||
path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'),
|
||||
);
|
||||
await fs.copyFile(
|
||||
path.join(root, 'scripts', 'install-agent-run-guard-agent.sh'),
|
||||
path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(runtimeRoot, 'RUNBOOK.txt'),
|
||||
[
|
||||
@@ -397,6 +405,9 @@ async function writeMetadata() {
|
||||
' node scripts/agent-run-worker.mjs --once',
|
||||
' bash scripts/install-agent-run-worker-agent.sh # installs disabled by default',
|
||||
' node scripts/check-agent-run-worker.mjs # read-only LaunchAgent/queue check',
|
||||
' node scripts/agent-run-guard.mjs # dry-run auto-pause guard check',
|
||||
' node scripts/agent-run-guard.mjs --apply # stop worker and disable code-run gate when thresholds trip',
|
||||
' bash scripts/install-agent-run-guard-agent.sh # installs guard LaunchAgent',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
@@ -425,6 +436,8 @@ async function main() {
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-runtime-heartbeat-agent.sh'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-runtime-slo-report-agent.sh'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-worker-agent.sh'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'agent-run-guard.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-agent-run-guard-agent.sh'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-agent-run-worker.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
|
||||
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||
SCRIPT="${MEMIND_AGENT_RUN_GUARD_SCRIPT:-$ROOT/scripts/agent-run-guard.mjs}"
|
||||
LABEL="${MEMIND_AGENT_RUN_GUARD_LABEL:-cn.tkmind.memind-agent-run-guard}"
|
||||
PLIST="$HOME/Library/LaunchAgents/${LABEL}.plist"
|
||||
LOG="${MEMIND_AGENT_RUN_GUARD_LOG:-$HOME/Library/Logs/memind-agent-run-guard.log}"
|
||||
GUI="gui/$(id -u)"
|
||||
INTERVAL="${MEMIND_AGENT_RUN_GUARD_INTERVAL_SECONDS:-60}"
|
||||
START="${MEMIND_AGENT_RUN_GUARD_START:-1}"
|
||||
|
||||
mkdir -p "$HOME/Library/LaunchAgents" "$(dirname "$LOG")"
|
||||
|
||||
if [[ ! -x "$NODE_BIN" ]]; then
|
||||
NODE_BIN="$(command -v node)"
|
||||
fi
|
||||
if [[ ! -f "$SCRIPT" ]]; then
|
||||
echo "agent run guard script not found: $SCRIPT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat > "$PLIST" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>$LABEL</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$NODE_BIN</string>
|
||||
<string>$SCRIPT</string>
|
||||
<string>--apply</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>$ROOT</string>
|
||||
<key>StartInterval</key>
|
||||
<integer>$INTERVAL</integer>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>$LOG</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>$LOG</string>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>PATH</key>
|
||||
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_ENABLED</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_ENABLED:-1}</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_FAILED_WINDOW_MS</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_FAILED_WINDOW_MS:-600000}</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_MAX_RECENT_FAILURES:-3}</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_MAX_PENDING_AGE_MS:-300000}</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_MAX_PENDING_COUNT:-10}</string>
|
||||
<key>MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS</key>
|
||||
<string>${MEMIND_AGENT_RUN_GUARD_MAX_RUNNING_AGE_MS:-900000}</string>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
plutil -lint "$PLIST"
|
||||
launchctl bootout "$GUI/$LABEL" 2>/dev/null || true
|
||||
launchctl bootstrap "$GUI" "$PLIST"
|
||||
|
||||
if [[ "$START" == "1" || "$START" == "true" || "$START" == "yes" ]]; then
|
||||
launchctl enable "$GUI/$LABEL"
|
||||
launchctl kickstart -k "$GUI/$LABEL"
|
||||
state="started"
|
||||
else
|
||||
launchctl disable "$GUI/$LABEL" 2>/dev/null || true
|
||||
state="installed-disabled"
|
||||
fi
|
||||
|
||||
echo "installed $PLIST"
|
||||
echo "state: $state"
|
||||
echo "script: $SCRIPT"
|
||||
echo "interval_seconds: $INTERVAL"
|
||||
echo "log: $LOG"
|
||||
echo "manual stop: launchctl bootout $GUI/$LABEL"
|
||||
Reference in New Issue
Block a user