Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 john
|
c247243b49 | ||
|
john
|
916641d748 | ||
|
john
|
78ca4aa12e | ||
|
john
|
1edec6e196 | ||
|
john
|
57d90e6113 | ||
|
john
|
3be5a04161 |
@@ -6,12 +6,6 @@
|
||||
#
|
||||
# 后端代理（server.mjs / vite dev proxy 共用）
|
||||
TKMIND_API_TARGET=https://127.0.0.1:18006
|
||||
# Native dual pool (local simulation): bash scripts/install-local-goosed-pool.sh
|
||||
# TKMIND_API_TARGET_1=https://127.0.0.1:18007
|
||||
# TKMIND_API_TARGETS=https://127.0.0.1:18006,https://127.0.0.1:18007
|
||||
# GOOSED_NATIVE_POOL_PORTS=18006,18007
|
||||
# GOOSED_RUNTIME=native # runtime-worker-metrics.mjs; auto detects when no docker pool
|
||||
# Soak: node scripts/soak-local-goosed-pool.mjs --minutes 30 --interval 60
|
||||
TKMIND_SERVER__SECRET_KEY=local-dev-secret
|
||||
H5_PORT=8081
|
||||
# Vite UI 预览用 5173；Agent 交付的 MindSpace 公开页 + Page Data API 走 Portal（8081）。
|
||||
|
||||
@@ -65,7 +65,8 @@ export const USER_FACING_ARCHITECTURE_TERM_REPLACEMENTS = Object.freeze([
|
||||
[/\b(?:stdio\s*)?mcp\b|\btkmind-search\b|\bsandbox-fs\b/gi, '工具扩展'],
|
||||
[/\bopenhands\b|\baider\b|\blitellm\b/gi, '代码助手'],
|
||||
[/\borchestrator\b|\bmemindadm\b|\bportal\b(?=\s*(?:api|server|runtime|8081|8085))/gi, '后台服务'],
|
||||
[/host\.docker\.internal|127\.0\.0\.1:\d{4,5}/gi, '本地服务'],
|
||||
// 仅脱敏非交付用途的 localhost；MindSpace 公开页链接（…/MindSpace/…）必须保留真实 host:port。
|
||||
[/host\.docker\.internal|(?:127\.0\.0\.1|localhost):\d{4,5}(?!\/MindSpace\/)/gi, '本地服务'],
|
||||
[/platform\/web|web_search|fetch_url|tkmind_search|tkmind_read/gi, '联网搜索'],
|
||||
[/\bpostgres(?:ql)?\b|\bredis\b|\bmysql\b|\bweaviate\b/gi, '数据服务'],
|
||||
]);
|
||||
|
||||
@@ -99,3 +99,12 @@ test('deriveAssistantFacingText keeps local MindSpace deliverable links clickabl
|
||||
'本地预览：http://127.0.0.1:5173/MindSpace/john/public/survey.html';
|
||||
assert.equal(deriveAssistantFacingText(localBare), localBare);
|
||||
});
|
||||
|
||||
test('deriveAssistantFacingText still redacts non-deliverable localhost references', () => {
|
||||
const internal =
|
||||
'Portal 后台服务监听在 127.0.0.1:8081，请稍后再试。';
|
||||
assert.equal(
|
||||
deriveAssistantFacingText(internal),
|
||||
'Portal 后台服务监听在 本地服务，请稍后再试。',
|
||||
);
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Local and 103 runtime topology
|
||||
|
||||
> Last confirmed: 2026-07-30 20:20 CST (goosed native migration).
|
||||
> Last confirmed: 2026-07-26 20:18 CST.
|
||||
>
|
||||
> This is the current topology source of truth for local Memind and 103 production. Prefer this document over older migration notes. Older architecture documents may contain historical paths from before the MindSpace split.
|
||||
|
||||
@@ -75,62 +75,48 @@ or whose `gitSha` does not match the runtime artifact manifest.
|
||||
|
||||
## goosed
|
||||
|
||||
103 goosed runs as **native launchd** processes (2026-07-30 migrated). Docker/Colima pool is stopped; `goosed-prod-1` is kept as emergency standby.
|
||||
|
||||
> 迁移记录与回滚：[goosed-native-103-migration-plan.md](./goosed-native-103-migration-plan.md)（**已执行 2026-07-30**）。脚本：`scripts/goosed-native-103-migrate.sh`。
|
||||
103 goosed runs in Colima/Docker, not as native launchd processes.
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Runtime type | Native launchd (`GOOSED_RUNTIME=native`) |
|
||||
| Root | `/Users/john/Project/tkmind_go-native` |
|
||||
| Binary | `tkmind_go-native/goosed` → `releases/goosed-*` |
|
||||
| Launchd labels | `cn.tkmind.goosed-native-18006` … `18014` |
|
||||
| Run script | `run-goosed-native.sh <port>` + `.env.<port>` |
|
||||
| Session store | PostgreSQL `memind_sessions` via `GOOSE_SESSION_DB_URL` → `127.0.0.1:5432` |
|
||||
| FD monitor | `cn.tkmind.goosed-monitor` → `scripts/monitor-goosed-fds.mjs` |
|
||||
| Docker standby | `goosed-prod-1` stopped, `restart=no`（紧急回滚 18006） |
|
||||
| DeepSeek no-think | `cn.tkmind.memind-deepseek-no-think` → `:18036`（native 必需，`MEMIND_DEEPSEEK_PROXY_ENTRYPOINT=1`） |
|
||||
| Docker context | `colima` |
|
||||
| Compose directory | `/Users/john/Project/goosed-prod` |
|
||||
| Compose env | `/Users/john/Project/goosed-prod/.env` |
|
||||
| Image pattern | `tkmind/goosed:prod-${GOOSED_TAG}` |
|
||||
| Session store | PostgreSQL `memind_sessions` through `GOOSE_SESSION_DB_URL` |
|
||||
| Session PostgreSQL | **Host** `postgresql@17` on `127.0.0.1:5432` (`/opt/homebrew/var/postgresql@17`) |
|
||||
|
||||
**Critical:** Native goosed uses host PostgreSQL on `127.0.0.1:5432`. If PostgreSQL stops, H5 `/agent/start` hangs ~60s. Recovery:
|
||||
**Critical:** Colima goosed reaches the session DB via `host.docker.internal:5432`. If host PostgreSQL is stopped, H5 `/agent/start` hangs ~60s then fails with `pool timed out`, and the UI stays on「正在创建新对话…」. Recovery:
|
||||
|
||||
```bash
|
||||
ssh john@58.38.22.103 'bash /Users/john/Project/Memind/scripts/ensure-goose-session-postgres.sh'
|
||||
ssh john@58.38.22.103 'bash /Users/john/Project/goosed-prod/scripts/ensure-goose-session-postgres.sh'
|
||||
ssh john@58.38.22.103 '/opt/homebrew/opt/postgresql@17/bin/pg_isready -h 127.0.0.1 -p 5432'
|
||||
```
|
||||
|
||||
Port layout (native listens directly on host port):
|
||||
`cn.tkmind.goosed-monitor`（103）运行 `goosed-prod/scripts/monitor-goosed-containers.mjs`，本地 native goosed 监控用 `scripts/monitor-goosed-fds.mjs`；两者都会在巡检时调用 `ensure-goose-session-postgres.sh`。
|
||||
|
||||
| Port | Launchd label |
|
||||
|------|---------------|
|
||||
| `18006` | `cn.tkmind.goosed-native-18006` |
|
||||
| `18007` | `cn.tkmind.goosed-native-18007` |
|
||||
| … | … |
|
||||
| `18014` | `cn.tkmind.goosed-native-18014` |
|
||||
Container layout (host port → container `18006`):
|
||||
|
||||
| Container | Host port | Container port |
|
||||
|-----------|-----------|----------------|
|
||||
| `goosed-prod-1` | `18006` | `18006` |
|
||||
| `goosed-prod-2` | `18007` | `18006` |
|
||||
| `goosed-prod-3` | `18008` | `18006` |
|
||||
| `goosed-prod-4` | `18009` | `18006` |
|
||||
| `goosed-prod-5` | `18010` | `18006` |
|
||||
| `goosed-prod-6` | `18011` | `18006` |
|
||||
| `goosed-prod-7` | `18012` | `18006` |
|
||||
| `goosed-prod-8` | `18013` | `18006` |
|
||||
| `goosed-prod-9` | `18014` | `18006` |
|
||||
|
||||
All nine host ports must return `ok` from `/status`, and must appear in Portal `.env` `TKMIND_API_TARGETS`.
|
||||
|
||||
### Legacy Docker pool (standby only)
|
||||
|
||||
| Item | Value |
|
||||
|------|-------|
|
||||
| Compose directory | `/Users/john/Project/goosed-prod` |
|
||||
| Image pattern | `tkmind/goosed:prod-${GOOSED_TAG}` |
|
||||
| Containers | `goosed-prod-1..9` — **stopped**, `docker update --restart=no` |
|
||||
|
||||
Do **not** leave `cn.tkmind.goosed-monitor` on `monitor-goosed-containers.mjs` with `GOOSED_RESTART_UNHEALTHY=1`; it will restart Docker and fight native ports.
|
||||
|
||||
```bash
|
||||
# Emergency: single Docker instance on 18006 only
|
||||
launchctl bootout gui/$(id -u)/cn.tkmind.goosed-native-18006
|
||||
docker update --restart=unless-stopped goosed-prod-1 && docker start goosed-prod-1
|
||||
```
|
||||
|
||||
## PostgreSQL rule
|
||||
|
||||
Both local native goosed and 103 native goosed persist sessions in PostgreSQL:
|
||||
Both local native goosed and 103 Colima goosed persist sessions in PostgreSQL:
|
||||
|
||||
- local native goosed: PostgreSQL database `goose_sessions_dev`
|
||||
- 103 native goosed: PostgreSQL database `memind_sessions`
|
||||
- 103 Colima goosed: PostgreSQL database `memind_sessions`
|
||||
|
||||
Do not assume a local SQLite session store when debugging current Memind/goosed behavior.
|
||||
|
||||
|
||||
@@ -1,365 +0,0 @@
|
||||
# 103 生产 goosed 迁移计划：Docker/Colima → Native 多实例
|
||||
|
||||
> **状态：** 已执行（2026-07-30 维护窗）
|
||||
> **适用范围：** 103（`58.38.22.103` / Mac Studio）生产环境
|
||||
> **目标运行时：** 去掉 goosed 对 Docker/Colima 的依赖，改为 native launchd 多实例（`18006`–`18014`）
|
||||
> **前提：** 允许维护窗口停机；**不要求**进行中 SSE/工具任务不断线
|
||||
> **关联拓扑：** [103-runtime-topology.md](./103-runtime-topology.md)
|
||||
> **发布闸门：** [production-release-guardian.md](./production-release-guardian.md)、[发包必看.md](./发包必看.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. 目标与边界
|
||||
|
||||
### 1.1 要达成什么
|
||||
|
||||
| 目标 | 说明 |
|
||||
|------|------|
|
||||
| 去掉 goosed Docker/Colima 池 | `goosed-prod-1..9` + canary 容器退役 |
|
||||
| 改为 native launchd 池 | 参考 103 已有 `tkmind_go-native` / 本机 `install-local-goosed-pool.sh` 模式，扩到 9 实例 |
|
||||
| 用户数据零丢失 | RDS、PostgreSQL、MindSpace 磁盘原样保留 |
|
||||
| 功能等价恢复 | 登录、聊天、续聊、页面生成、微信、Plaza/adm 等与迁移前一致 |
|
||||
| 消除 bind mount inode 漂移 | Portal 发版后不再依赖 `docker compose force-recreate` remount |
|
||||
|
||||
### 1.2 「无缝」的定义（本计划）
|
||||
|
||||
| 维度 | 本计划承诺 | 不承诺 |
|
||||
|------|------------|--------|
|
||||
| 账号、余额、MindSpace 页面、历史落库 | 零丢失 | — |
|
||||
| 系统功能 | 维护窗结束后等价恢复 | — |
|
||||
| 进行中的 SSE / 工具执行 | — | 不断线；用户需刷新或重开对话 |
|
||||
| 零停机 | — | 必须安排维护窗口 |
|
||||
|
||||
架构约束（[memind-2-streaming-agent-runtime-plan.md](./architecture/memind-2-streaming-agent-runtime-plan.md)）：
|
||||
|
||||
- 已存在 session **保持 worker affinity，不做中途迁移**
|
||||
- **不做无损中途迁移**，除非 goosed 支持完整 session restore
|
||||
|
||||
### 1.3 不在本次范围
|
||||
|
||||
- 103 迁到新物理机
|
||||
- 实例数从 9 缩到 2（首次迁移禁止同时缩容）
|
||||
- 修改 RDS / 用户 schema
|
||||
- 合并尚未走完发布闸门的 Portal 大功能（除非维护窗内单独批准发版）
|
||||
|
||||
---
|
||||
|
||||
## 2. 生产数据与依赖地图
|
||||
|
||||
```text
|
||||
105 nginx (m.tkmind.cn)
|
||||
│
|
||||
▼
|
||||
103 Portal :8081 cn.tkmind.memind-portal
|
||||
│
|
||||
├── 阿里云 RDS MySQL `goose` 用户、计费、h5_user_sessions.goosed_target、agent_run…
|
||||
├── MindSpace Service :8082 /Users/john/MindSpace（不动）
|
||||
├── memind_adm :8085 （不动）
|
||||
├── Plaza :3001 （不动）
|
||||
│
|
||||
├── goosed 18006–18014 【本次替换】Docker → native
|
||||
│ └── PG5432 memind_sessions（共享，不迁移数据）
|
||||
│
|
||||
├── PG5433 mindspace_userdata_prod
|
||||
├── Redis :6379 运行时路由/指标（可清空重建）
|
||||
└── 磁盘（Portal persist）
|
||||
.env
|
||||
/Users/john/MindSpace/data/mindspace
|
||||
/Users/john/Project/Memind/users
|
||||
/Users/john/Project/Memind/data
|
||||
public/plaza-covers, logs, .tailscale
|
||||
```
|
||||
|
||||
### 2.1 持久化清单（必须备份）
|
||||
|
||||
与 `scripts/release-portal-runtime-prod.sh` manifest 一致：
|
||||
|
||||
```text
|
||||
.env, MindSpace, data, users, .tailscale, public/plaza-covers, logs
|
||||
```
|
||||
|
||||
**额外必须备份（不在 persist tar 内）：**
|
||||
|
||||
| 资产 | 备份方式 |
|
||||
|------|----------|
|
||||
| 阿里云 RDS `goose` | 控制台快照 + 可选逻辑导出 |
|
||||
| PostgreSQL `memind_sessions` | `pg_dump` |
|
||||
| PostgreSQL `mindspace_userdata_prod` | `pg_dump` |
|
||||
| `goosed-prod` compose + `.env` + 镜像 tag | 目录 tar + `GOOSED_TAG` 记录 |
|
||||
| `tkmind_go-native`（若已存在） | 目录拷贝 |
|
||||
| Portal `.release-manifest.txt` | 记录 `release_id` / `git_head` |
|
||||
| 105 nginx `m.tkmind.cn` 等配置 | 发布脚本同款备份 |
|
||||
|
||||
### 2.2 Session 与流量（迁移后行为不变）
|
||||
|
||||
- **新对话：** Portal `pickTarget()` 在 `TKMIND_API_TARGETS` 上轮询 / Redis 负载评分
|
||||
- **老对话：** MySQL `h5_user_sessions.goosed_target` + Redis `session:{id}:target` 粘住原 `https://127.0.0.1:1800N`
|
||||
- **goosed 会话体：** PG `memind_sessions` 多实例共享；native 直连 `127.0.0.1:5432`（替代 Docker `host.docker.internal`）
|
||||
|
||||
**首次迁移硬规则：native 仍监听 `18006`–`18014` 全端口**，与现网 `goosed_target` URL 兼容。
|
||||
|
||||
---
|
||||
|
||||
## 3. 迁移前准入条件（Gate）
|
||||
|
||||
以下 **全部满足** 才允许申请 103 维护窗：
|
||||
|
||||
### 3.1 本机验证（不动 103）
|
||||
|
||||
| # | 项 | 命令 / 标准 |
|
||||
|---|-----|-------------|
|
||||
| 1 | native 双实例 soak | `node scripts/soak-local-goosed-pool.mjs --minutes 30 --interval 60` 通过 |
|
||||
| 2 | 池健康检查 | `node scripts/check-local-goosed-pool.mjs` |
|
||||
| 3 | Portal 多 target | `.env` 中 `TKMIND_API_TARGETS=18006,18007` + 续聊 / 新建 session 冒烟 |
|
||||
| 4 | native 指标 | `GOOSED_RUNTIME=native node scripts/runtime-worker-metrics.mjs status --dry-run` |
|
||||
| 5 | 安装脚本 | `bash scripts/install-local-goosed-pool.sh` 可重复执行 |
|
||||
| 6 | 发布脚本 native 分支 | `GOOSED_RUNTIME=native` 时 `release-portal-runtime-prod.sh` 不依赖 Docker remount（合并 main 后） |
|
||||
| 7 | 回滚演练 | 本机或 103 只读：Docker compose 停/起 + `/status` 全绿 |
|
||||
|
||||
### 3.2 仓库与发布
|
||||
|
||||
| # | 项 |
|
||||
|---|-----|
|
||||
| 1 | 相关脚本 / 文档已合并 `main`，CI 通过 |
|
||||
| 2 | `bash scripts/check-release-ready.sh` 通过 |
|
||||
| 3 | 生产发布守门员 Core + Impact Gate report 已准备（与维护窗发版 commit 绑定） |
|
||||
| 4 | 用户 **明确批准** 103 维护窗与迁移执行（口头「继续」不构成批准） |
|
||||
|
||||
### 3.3 103 只读基线（迁移前 24h 内采集）
|
||||
|
||||
```bash
|
||||
# 只读，禁止改 103
|
||||
ssh john@58.38.22.103 'cat /Users/john/Project/Memind/.release-manifest.txt'
|
||||
ssh john@58.38.22.103 'grep ^TKMIND_API_TARGETS= /Users/john/Project/Memind/.env'
|
||||
ssh john@58.38.22.103 'grep ^GOOSED_TAG= /Users/john/Project/goosed-prod/.env'
|
||||
ssh john@58.38.22.103 'for p in $(seq 18006 18014); do curl -kfsS --connect-timeout 2 https://127.0.0.1:$p/status; echo; done'
|
||||
ssh john@58.38.22.103 'curl -fsS http://127.0.0.1:8082/mindspace/v1/contract | head -c 400'
|
||||
```
|
||||
|
||||
保存到本机 `test/_103_baselines/pre-native-migration-YYYYMMDD/`。
|
||||
|
||||
---
|
||||
|
||||
## 4. 维护窗口执行计划
|
||||
|
||||
**建议窗口：** ≥ 30 分钟（9 实例安装 + provider 同步 + 回归）
|
||||
**建议时段：** 低峰 + 可联系 105 改 nginx 的人员在场
|
||||
|
||||
### Phase 0 — 备份（T0 前，可先执行）
|
||||
|
||||
- [ ] RDS 快照
|
||||
- [ ] `pg_dump memind_sessions`、`mindspace_userdata_prod`
|
||||
- [ ] 103 上执行 persist 备份（或等价 `memind-persisted-*-before.tar.gz`）
|
||||
- [ ] 全目录 / compose / 105 nginx 备份
|
||||
- [ ] 记录：`release_id`、`GOOSED_TAG`、九个 Docker 容器 ID、`TKMIND_API_TARGETS`
|
||||
|
||||
### Phase 1 — 切流量（停机开始）
|
||||
|
||||
- [ ] 105：维护页或 upstream 指向维护（禁止新用户进入）
|
||||
- [ ] 103：可选公告 / 微信模板（「系统维护约 N 分钟」）
|
||||
- [ ] **Soft drain（推荐，最多等 10 分钟）：**
|
||||
|
||||
```bash
|
||||
# 103 上，对每个 worker 设 drain，等待 active_streams 归零
|
||||
# 见 memind-2-streaming-agent-runtime-plan.md P5 drain 说明
|
||||
docker exec memind-runtime-redis redis-cli SET memind:runtime:worker:goosed-N:drain 1
|
||||
curl -sk https://m.tkmind.cn/api/runtime/status # 观察 activeStreams
|
||||
```
|
||||
|
||||
- [ ] 超时后 **硬停**：`cn.tkmind.memind-portal`（或仅阻止新 session，视 drain 结果）
|
||||
|
||||
### Phase 2 — 替换 goosed 运行时
|
||||
|
||||
- [ ] 停止 Docker goosed 池：
|
||||
|
||||
```bash
|
||||
cd /Users/john/Project/goosed-prod
|
||||
docker compose -f docker-compose.prod.yml down
|
||||
docker rm -f goosed-prod-canary 2>/dev/null || true
|
||||
```
|
||||
|
||||
- [ ] 部署 native 9 实例（目录模式参考 `tkmind_go-native`，端口 `18006`–`18014`）：
|
||||
- 同一 `GOOSE_SESSION_DB_URL` → `memind_sessions`
|
||||
- `GOOSE_TLS=true`
|
||||
- `GOOSED_MCP_CONTAINERIZED=0`
|
||||
- `GOOSED_MCP_*` 使用宿主机路径（与 Portal `.env` 一致）
|
||||
- `MINDSPACE_STORAGE_ROOT=/Users/john/MindSpace/data/mindspace`
|
||||
- `PORTAL_RUNTIME_DIR=/Users/john/Project/Memind`
|
||||
|
||||
- [ ] 更新 Portal `.env`（若需要）：
|
||||
- `TKMIND_API_TARGETS=https://127.0.0.1:18006,...,https://127.0.0.1:18014`（**含 18006**）
|
||||
- `GOOSED_RUNTIME=native`
|
||||
- DeepSeek no-think：`127.0.0.1:18036`（去掉 `host.docker.internal` 依赖）
|
||||
|
||||
- [ ] **MindSpace :8082、RDS、磁盘：不移动、不删**
|
||||
|
||||
- [ ] Provider 同步到 **全部 9 个 target**
|
||||
|
||||
- [ ] 验收九个 `/status`：
|
||||
|
||||
```bash
|
||||
for p in $(seq 18006 18014); do
|
||||
echo -n "$p: "
|
||||
curl -kfsS --connect-timeout 3 "https://127.0.0.1:$p/status" || echo FAIL
|
||||
done
|
||||
```
|
||||
|
||||
### Phase 3 — Portal 与附属服务
|
||||
|
||||
**若维护窗内无 Portal 代码变更：**
|
||||
|
||||
- [ ] 重启 `cn.tkmind.memind-portal`（读新 `.env`）
|
||||
- [ ] **跳过** Docker remount 步骤
|
||||
|
||||
**若维护窗内包含 Portal runtime 发版：**
|
||||
|
||||
- [ ] 按 [发包必看.md](./发包必看.md) 走 canary 或晋升流程
|
||||
- [ ] 发版脚本使用 `GOOSED_RUNTIME=native` 检查链
|
||||
- [ ] MindSpace contract `gitSha` 与 manifest 对齐
|
||||
|
||||
**始终确认仍在运行：**
|
||||
|
||||
- [ ] `cn.tkmind.mindspace-service` :8082
|
||||
- [ ] memind_adm :8085、Plaza :3001、imgproxy、SearXNG、agent-run-worker、DeepSeek no-think :18036
|
||||
|
||||
### Phase 4 — 功能回归（103 localhost + 105 外网）
|
||||
|
||||
| 优先级 | 场景 | 验证方式 |
|
||||
|--------|------|----------|
|
||||
| P0 | Portal `/api/status` | `curl http://127.0.0.1:8081/api/status` |
|
||||
| P0 | 登录 / 鉴权 | `/auth/login`、`/auth/status` |
|
||||
| P0 | 新建对话 | H5 开聊，检查 `h5_user_sessions.goosed_target` 分布 |
|
||||
| P0 | **旧对话续聊** | 维护前存在的 session，迁移后 `/reply` 成功 |
|
||||
| P0 | 页面列表 / 公开页 | MindSpace remote + 已有 HTML 可访问 |
|
||||
| P1 | page.generate / edit_file | workspace 落盘 + `public/*.html` |
|
||||
| P1 | 微信真实回调 | 非仅 health |
|
||||
| P1 | MindSearch / 图片 imgproxy | 生产 URL |
|
||||
| P1 | agent-run worker | 队列消费 |
|
||||
| P2 | Plaza、adm 后台 | 各端口 health |
|
||||
|
||||
生产 Gate：维护窗发版 commit 须绑定 **Core + Impact Gate report**（见 production-release-guardian）。
|
||||
|
||||
### Phase 5 — 恢复流量
|
||||
|
||||
- [ ] 105 nginx 恢复 `58.38.22.103:8081`（或既定 upstream）
|
||||
- [ ] 外网抽样：`https://m.tkmind.cn/api/status`
|
||||
- [ ] 维护结束公告
|
||||
|
||||
### Phase 6 — 迁移后观察（24–72h）
|
||||
|
||||
- [ ] `GOOSED_RUNTIME=native node scripts/runtime-worker-metrics.mjs sample`
|
||||
- [ ] native FD 监控（`monitor-goosed-fds.mjs`，覆盖 `18006`–`18014`）
|
||||
- [ ] Redis worker 指标、SLO report 无异常尖刺
|
||||
- [ ] 用户续聊 / 页面生成无集中投诉
|
||||
|
||||
---
|
||||
|
||||
## 5. 回滚计划
|
||||
|
||||
**触发条件：** P0 回归失败、九个 target 无法全绿、旧 session 批量无法续聊、FD 泄漏导致整机不稳定
|
||||
|
||||
**目标时间：** 维护窗内 < 15 分钟完成回滚
|
||||
|
||||
| 步骤 | 操作 |
|
||||
|------|------|
|
||||
| 1 | 105 维护页保持 |
|
||||
| 2 | 停 native launchd 池（`18006`–`18014`） |
|
||||
| 3 | `cd goosed-prod && docker compose -f docker-compose.prod.yml up -d`（**原 GOOSED_TAG**） |
|
||||
| 4 | `.env` 恢复备份：`TKMIND_API_TARGETS`、Docker MCP 路径、`GOOSED_RUNTIME=docker` |
|
||||
| 5 | 若 Portal 已发新版：回滚至备份 `release_id` runtime |
|
||||
| 6 | 九容器 healthy + provider 同步 + `/status` 全绿 |
|
||||
| 7 | 105 恢复流量 |
|
||||
|
||||
**回滚不需要：** 恢复 RDS / PG（迁移中若未改库）。
|
||||
|
||||
---
|
||||
|
||||
## 6. 配置对照表（Docker → Native）
|
||||
|
||||
| 配置项 | Docker 生产（现） | Native 生产（目标） |
|
||||
|--------|-------------------|---------------------|
|
||||
| goosed 进程 | `goosed-prod-1..9` 容器 | `com.tkmind.goosed-native-1800N` launchd |
|
||||
| Session PG | `host.docker.internal:5432` | `127.0.0.1:5432` |
|
||||
| MCP | 容器内 `/usr/local/bin/node` 等 | 宿主机 `GOOSED_MCP_*` |
|
||||
| `GOOSED_MCP_CONTAINERIZED` | 隐式容器 | `0` 或不设 |
|
||||
| Portal remount | 发版后 `compose force-recreate` | **不需要** |
|
||||
| 监控 | `monitor-goosed-containers.mjs` | `monitor-goosed-fds.mjs`（扩展全端口） |
|
||||
| 指标 | `runtime-worker-metrics` docker | `GOOSED_RUNTIME=native` |
|
||||
| Canary goosed | `goosed-prod-canary:18015` | native `18015`（Phase 2 后单独迁移） |
|
||||
|
||||
---
|
||||
|
||||
## 7. 风险与缓解
|
||||
|
||||
| 风险 | 严重度 | 缓解 |
|
||||
|------|--------|------|
|
||||
| 进行中对话全部中断 | 高 | 维护公告；soft drain；恢复后提示刷新 |
|
||||
| 旧 session PG restore 失败 | 高 | 迁移前用 PG 副本在 staging 验续聊 |
|
||||
| provider 未覆盖全 target | 高 | 自动化检查 + 发版脚本 gate |
|
||||
| MCP 路径 / sandbox 失败 | 高 | `page.generate` smoke + sandbox-fs |
|
||||
| `goosed_target` 指向已下线端口 | 中 | 首次迁移保持 9 端口不变 |
|
||||
| native FD 泄漏 ×9 | 中 | 72h 监控 + kickstart 阈值 |
|
||||
| MindSpace / Portal 版本漂移 | 中 | contract gitSha 对账 |
|
||||
| 105 切流后 CDN/缓存 | 低 | 外网真实 URL 验证 |
|
||||
|
||||
---
|
||||
|
||||
## 8. 时间线模板
|
||||
|
||||
| 时点 | 动作 | 环境 |
|
||||
|------|------|------|
|
||||
| T-14d | 本机 native 双实例 → 脚本 native 化完成 | 本机 |
|
||||
| T-7d | 本机 native 9 实例 soak（若与生产同构） | 本机 |
|
||||
| T-3d | 103 只读基线 + 备份演练 + Docker 回滚演练 | 103 只读/演练 |
|
||||
| T-1d | 维护公告定稿；105 维护页脚本就绪 | 105 |
|
||||
| T0 | Phase 0–5 执行 | 103 + 105 |
|
||||
| T+24h | FD / 续聊 / 微信观察 | 103 |
|
||||
| T+7d | 评审：是否退役 Colima、删 Docker 镜像、更新 topology 文档 | 文档 |
|
||||
| T+30d | 可选：评估实例缩容（**单独变更**，不在本计划内） | — |
|
||||
|
||||
---
|
||||
|
||||
## 9. 迁移后文档更新
|
||||
|
||||
迁移成功并稳定 7 天后：
|
||||
|
||||
- [ ] 更新 [103-runtime-topology.md](./103-runtime-topology.md)（goosed 段改为 native 9 实例）
|
||||
- [ ] 更新 [发包必看.md](./发包必看.md) §5 goosed 镜像 → native 二进制发布
|
||||
- [ ] 在 [docs/branch-disposition.md](./branch-disposition.md) 登记相关功能分支（若适用）
|
||||
- [ ] 归档 Docker compose 回滚包位置与 `GOOSED_TAG`
|
||||
|
||||
---
|
||||
|
||||
## 10. 本仓库工具索引
|
||||
|
||||
| 用途 | 路径 |
|
||||
|------|------|
|
||||
| 本机安装 native 池 | `bash scripts/install-local-goosed-pool.sh` |
|
||||
| 池健康检查 | `node scripts/check-local-goosed-pool.mjs` |
|
||||
| 长跑 soak | `node scripts/soak-local-goosed-pool.mjs` |
|
||||
| native worker 指标 | `GOOSED_RUNTIME=native node scripts/runtime-worker-metrics.mjs` |
|
||||
| FD 监控 | `scripts/monitor-goosed-fds.mjs` + `scripts/install-goosed-monitor.sh` |
|
||||
| Portal 生产发布 | `scripts/release-portal-runtime-prod.sh` |
|
||||
| 发布就绪 | `bash scripts/check-release-ready.sh` |
|
||||
|
||||
---
|
||||
|
||||
## 11. 批准记录（执行时填写）
|
||||
|
||||
| 字段 | 值 |
|
||||
|------|-----|
|
||||
| 维护窗口 | YYYY-MM-DD HH:MM – HH:MM (UTC+8) |
|
||||
| 执行人 | |
|
||||
| Portal `git_head` | |
|
||||
| Native goosed 二进制 / release | |
|
||||
| 回滚 `GOOSED_TAG` | |
|
||||
| RDS 快照 ID | |
|
||||
| persist_backup 路径 | |
|
||||
| Gate report ID | |
|
||||
| 结果 | 成功 / 回滚 |
|
||||
| 备注 | |
|
||||
|
||||
---
|
||||
|
||||
**版本：** 2026-07-30 v1
|
||||
**下次评审：** 本机 Phase 3（发布脚本 native 分支）合并 main 后
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.join(__dirname, '..');
|
||||
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.flatMap((value) => (
|
||||
value.startsWith('http')
|
||||
? [value]
|
||||
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
|
||||
));
|
||||
|
||||
if (targets.length === 0) {
|
||||
console.error('No targets configured. Set TKMIND_API_TARGETS or GOOSED_NATIVE_POOL_PORTS.');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const target of targets) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const res = await undiciFetch(`${target.replace(/\/$/, '')}/status`, {
|
||||
dispatcher: insecureDispatcher,
|
||||
});
|
||||
const body = (await res.text()).trim();
|
||||
results.push({
|
||||
target,
|
||||
ok: res.ok && body === 'ok',
|
||||
status: res.status,
|
||||
body,
|
||||
ms: Date.now() - started,
|
||||
});
|
||||
} catch (err) {
|
||||
results.push({
|
||||
target,
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
ms: Date.now() - started,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: results.every((item) => item.ok),
|
||||
count: results.length,
|
||||
results,
|
||||
}, null, 2));
|
||||
|
||||
process.exit(results.every((item) => item.ok) ? 0 : 1);
|
||||
@@ -1,445 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# 103 production: Docker goosed pool -> native launchd pool (18006-18014).
|
||||
# Keeps goosed-prod-1 container stopped as Docker hot standby.
|
||||
#
|
||||
# Usage on 103:
|
||||
# bash scripts/goosed-native-103-migrate.sh backup
|
||||
# bash scripts/goosed-native-103-migrate.sh migrate
|
||||
# bash scripts/goosed-native-103-migrate.sh verify
|
||||
# bash scripts/goosed-native-103-migrate.sh rollback # emergency Docker standby on 18006 only
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="/opt/homebrew/bin:/usr/local/bin:${PATH}"
|
||||
|
||||
ROOT="${MEMIND_ROOT:-/Users/john/Project/Memind}"
|
||||
GOOSED_NATIVE_ROOT="${GOOSED_NATIVE_ROOT:-/Users/john/Project/tkmind_go-native}"
|
||||
GOOSED_PROD_ROOT="${GOOSED_PROD_ROOT:-/Users/john/Project/goosed-prod}"
|
||||
PORTS=(18006 18007 18008 18009 18010 18011 18012 18013 18014)
|
||||
DOCKER_STANDBY="goosed-prod-1"
|
||||
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
|
||||
GUI="gui/$(id -u)"
|
||||
BACKUP_ROOT="${ROOT}/backups/goosed-native-migration"
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/pre-native-${STAMP}"
|
||||
|
||||
log() { printf '[goosed-native-103] %s\n' "$*"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
require_103() {
|
||||
[[ "$(hostname -s 2>/dev/null || hostname)" == "johndeMac-Studio" || "$(hostname)" == *Studio* ]] || true
|
||||
[[ -d "${ROOT}" ]] || die "Memind root not found: ${ROOT}"
|
||||
}
|
||||
|
||||
free_disk_headroom() {
|
||||
log "Rotating oversized logs to free disk headroom before backup"
|
||||
for f in \
|
||||
"${HOME}/Library/Logs/memind-agent-run-worker.log" \
|
||||
"${HOME}/Library/Logs/memind-runtime-heartbeat.log" \
|
||||
"${HOME}/Library/Logs/memind-frontend-5173.log"; do
|
||||
if [[ -f "${f}" ]] && [[ "$(wc -c <"${f}" | tr -d ' ')" -gt 50000000 ]]; then
|
||||
mv "${f}" "${f}.bak-${STAMP}"
|
||||
: >"${f}"
|
||||
log "rotated ${f}"
|
||||
fi
|
||||
done
|
||||
df -h "${ROOT}" | tail -1
|
||||
}
|
||||
|
||||
phase_backup() {
|
||||
require_103
|
||||
free_disk_headroom
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
log "Backup directory: ${BACKUP_DIR}"
|
||||
|
||||
{
|
||||
echo "stamp=${STAMP}"
|
||||
echo "hostname=$(hostname)"
|
||||
date -Iseconds
|
||||
cat "${ROOT}/.release-manifest.txt" 2>/dev/null || true
|
||||
grep '^TKMIND_API_TARGETS=' "${ROOT}/.env" || true
|
||||
grep '^GOOSED_TAG=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null || true
|
||||
docker ps -a --format '{{.Names}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}' | grep goosed || true
|
||||
} >"${BACKUP_DIR}/baseline.txt"
|
||||
|
||||
cp "${ROOT}/.env" "${BACKUP_DIR}/portal.env"
|
||||
cp "${GOOSED_PROD_ROOT}/.env" "${BACKUP_DIR}/goosed-prod.env" 2>/dev/null || true
|
||||
cp "${GOOSED_PROD_ROOT}/docker-compose.prod.yml" "${BACKUP_DIR}/" 2>/dev/null || true
|
||||
|
||||
log "pg_dump memind_sessions"
|
||||
local session_url
|
||||
session_url="$(grep '^GOOSE_SESSION_DB_URL=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null | cut -d= -f2- | sed 's|@host.docker.internal|@127.0.0.1|' || true)"
|
||||
if [[ -z "${session_url}" ]]; then
|
||||
session_url="postgresql://boot:@Abc888888@127.0.0.1:5432/memind_sessions"
|
||||
fi
|
||||
pg_dump "${session_url}" | gzip -1 >"${BACKUP_DIR}/memind_sessions.sql.gz"
|
||||
|
||||
log "pg_dump mindspace_userdata_prod"
|
||||
local userdata_url
|
||||
userdata_url="$(grep '^MINDSPACE_USERDATA_PG_URL=' "${ROOT}/.env" | cut -d= -f2-)"
|
||||
pg_dump "${userdata_url}" | gzip -1 >"${BACKUP_DIR}/mindspace_userdata_prod.sql.gz"
|
||||
|
||||
log "persisted assets tar"
|
||||
tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
|
||||
-C "${ROOT}" \
|
||||
.env MindSpace data users .tailscale public/plaza-covers logs 2>/dev/null \
|
||||
|| tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
|
||||
-C "${ROOT}" .env data users logs 2>/dev/null
|
||||
|
||||
log "MindSpace storage tar"
|
||||
tar -czf "${BACKUP_DIR}/mindspace-storage-${STAMP}.tar.gz" \
|
||||
-C /Users/john/MindSpace data/mindspace 2>/dev/null || true
|
||||
|
||||
log "goosed-prod directory tar"
|
||||
tar -czf "${BACKUP_DIR}/goosed-prod-${STAMP}.tar.gz" \
|
||||
-C "${GOOSED_PROD_ROOT}" .env docker-compose.prod.yml README.md 2>/dev/null || true
|
||||
|
||||
log "tkmind_go-native directory tar"
|
||||
tar -czf "${BACKUP_DIR}/tkmind_go-native-${STAMP}.tar.gz" \
|
||||
-C "${GOOSED_NATIVE_ROOT}" \
|
||||
.env.18006 run-goosed-18006.sh deploy releases goosed 2>/dev/null || true
|
||||
|
||||
log "docker inspect goosed containers"
|
||||
docker ps -aq --filter 'name=goosed-prod' | xargs docker inspect >"${BACKUP_DIR}/docker-goosed-inspect.json" 2>/dev/null || true
|
||||
|
||||
log "full Memind source tar (may take a few minutes)"
|
||||
tar -czf "${BACKUP_DIR}/memind-full-${STAMP}.tar.gz" \
|
||||
--exclude='./node_modules' \
|
||||
--exclude='./backups' \
|
||||
--exclude='./.git' \
|
||||
-C "${ROOT}" . 2>/dev/null || log "WARN: full tar skipped or partial — check disk space"
|
||||
|
||||
ln -sfn "${BACKUP_DIR}" "${BACKUP_ROOT}/latest"
|
||||
du -sh "${BACKUP_DIR}"/*
|
||||
df -h "${ROOT}" | tail -1
|
||||
log "Backup complete: ${BACKUP_DIR}"
|
||||
}
|
||||
|
||||
install_native_env_files() {
|
||||
local port base="${GOOSED_NATIVE_ROOT}/.env.18006"
|
||||
[[ -f "${base}" ]] || die "missing ${base}"
|
||||
for port in "${PORTS[@]}"; do
|
||||
local env_file="${GOOSED_NATIVE_ROOT}/.env.${port}"
|
||||
if [[ ! -f "${env_file}" ]]; then
|
||||
sed "s/^GOOSE_PORT=.*/GOOSE_PORT=${port}/" "${base}" >"${env_file}"
|
||||
log "created ${env_file}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
install_native_run_script() {
|
||||
local script="${GOOSED_NATIVE_ROOT}/run-goosed-native.sh"
|
||||
cat >"${script}" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
PORT="${1:?missing port argument}"
|
||||
ROOT="/Users/john/Project/tkmind_go-native"
|
||||
cd "${ROOT}"
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "${ROOT}/.env.${PORT}"
|
||||
set +a
|
||||
exec "${ROOT}/goosed" agent
|
||||
EOF
|
||||
chmod +x "${script}"
|
||||
}
|
||||
|
||||
install_native_launchd() {
|
||||
local port label plist
|
||||
install_native_run_script
|
||||
for port in "${PORTS[@]}"; do
|
||||
label="cn.tkmind.goosed-native-${port}"
|
||||
plist="${LAUNCHD_DIR}/${label}.plist"
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>${label}</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>${GOOSED_NATIVE_ROOT}/run-goosed-native.sh</string>
|
||||
<string>${port}</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${GOOSED_NATIVE_ROOT}</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
launchctl enable "${GUI}/${label}"
|
||||
launchctl kickstart -k "${GUI}/${label}" 2>/dev/null || launchctl kickstart "${GUI}/${label}"
|
||||
log "launchd ${label} installed"
|
||||
done
|
||||
}
|
||||
|
||||
phase_start_deepseek_proxy() {
|
||||
local plist="${LAUNCHD_DIR}/cn.tkmind.memind-deepseek-no-think.plist"
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key><string>cn.tkmind.memind-deepseek-no-think</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/opt/homebrew/opt/node@24/bin/node</string>
|
||||
<string>${ROOT}/deepseek-no-think-proxy.mjs</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>MEMIND_DEEPSEEK_PROXY_ENTRYPOINT</key><string>1</string>
|
||||
<key>MEMIND_DEEPSEEK_DISABLE_THINKING</key><string>1</string>
|
||||
<key>MEMIND_DEEPSEEK_NO_THINK_PORT</key><string>18036</string>
|
||||
<key>MEMIND_DEEPSEEK_NO_THINK_HOST</key><string>127.0.0.1</string>
|
||||
</dict>
|
||||
<key>WorkingDirectory</key><string>${ROOT}</string>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
|
||||
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-compat-candidate" 2>/dev/null || true
|
||||
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-no-think" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
curl -fsS "http://127.0.0.1:18036/health" >/dev/null || die "DeepSeek no-think proxy :18036 failed"
|
||||
log "DeepSeek no-think proxy listening on :18036"
|
||||
}
|
||||
|
||||
phase_reconfigure_monitor() {
|
||||
local plist="${LAUNCHD_DIR}/cn.tkmind.goosed-monitor.plist"
|
||||
local monitor_script="${ROOT}/scripts/monitor-goosed-fds.mjs"
|
||||
[[ -f "${monitor_script}" ]] || die "missing ${monitor_script}"
|
||||
cp "${plist}" "${plist}.bak-docker-${STAMP}" 2>/dev/null || true
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>GOOSED_NATIVE_POOL_PORTS</key>
|
||||
<string>18006,18007,18008,18009,18010,18011,18012,18013,18014</string>
|
||||
<key>GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE</key>
|
||||
<string>cn.tkmind.goosed-native-{port}</string>
|
||||
<key>GOOSED_FD_RESTART</key>
|
||||
<string>3200</string>
|
||||
<key>GOOSED_FD_WARN</key>
|
||||
<string>180</string>
|
||||
<key>PATH</key>
|
||||
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||
<key>SANDBOX_MCP_MAX_PER_ROOT</key>
|
||||
<string>2</string>
|
||||
<key>SANDBOX_MCP_MAX_TOTAL</key>
|
||||
<string>80</string>
|
||||
</dict>
|
||||
<key>Label</key>
|
||||
<string>cn.tkmind.goosed-monitor</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/opt/homebrew/opt/node@24/bin/node</string>
|
||||
<string>${monitor_script}</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
|
||||
<key>StartInterval</key>
|
||||
<integer>60</integer>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${ROOT}</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/cn.tkmind.goosed-monitor" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
log "goosed-monitor switched to native FD monitor"
|
||||
}
|
||||
|
||||
stop_docker_pool_keep_standby() {
|
||||
local name
|
||||
for name in goosed-prod-{1..9}; do
|
||||
docker update --restart=no "${name}" 2>/dev/null || true
|
||||
done
|
||||
for name in goosed-prod-{2..9}; do
|
||||
if docker ps -q --filter "name=^/${name}$" | grep -q .; then
|
||||
log "stopping docker ${name}"
|
||||
docker stop "${name}" >/dev/null
|
||||
fi
|
||||
done
|
||||
if docker ps -q --filter "name=^/${DOCKER_STANDBY}$" | grep -q .; then
|
||||
log "stopping docker standby ${DOCKER_STANDBY} (container kept, restart=no)"
|
||||
docker stop "${DOCKER_STANDBY}" >/dev/null
|
||||
fi
|
||||
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}' || true
|
||||
}
|
||||
|
||||
update_portal_env() {
|
||||
local env_file="${ROOT}/.env"
|
||||
local targets=""
|
||||
local port
|
||||
for port in "${PORTS[@]}"; do
|
||||
targets="${targets}https://127.0.0.1:${port},"
|
||||
done
|
||||
targets="${targets%,}"
|
||||
|
||||
cp "${env_file}" "${env_file}.bak-native-${STAMP}"
|
||||
|
||||
if grep -q '^GOOSED_RUNTIME=' "${env_file}"; then
|
||||
sed -i '' 's/^GOOSED_RUNTIME=.*/GOOSED_RUNTIME=native/' "${env_file}"
|
||||
else
|
||||
printf '\nGOOSED_RUNTIME=native\n' >>"${env_file}"
|
||||
fi
|
||||
|
||||
sed -i '' "s|^TKMIND_API_TARGETS=.*|TKMIND_API_TARGETS=${targets}|" "${env_file}"
|
||||
sed -i '' 's|^TKMIND_API_TARGET=.*|TKMIND_API_TARGET=https://127.0.0.1:18006|' "${env_file}"
|
||||
|
||||
if grep -q '^GOOSED_MCP_NODE_PATH=' "${env_file}"; then
|
||||
sed -i '' 's|^GOOSED_MCP_NODE_PATH=.*|GOOSED_MCP_NODE_PATH=/opt/homebrew/opt/node@24/bin/node|' "${env_file}"
|
||||
fi
|
||||
if grep -q '^MEMIND_GOOSED_HOST_GATEWAY=' "${env_file}"; then
|
||||
sed -i '' 's|^MEMIND_GOOSED_HOST_GATEWAY=.*|MEMIND_GOOSED_HOST_GATEWAY=127.0.0.1|' "${env_file}"
|
||||
fi
|
||||
if grep -q '^TKMIND_SEARCH_MCP_HOST_GATEWAY=' "${env_file}"; then
|
||||
sed -i '' 's|^TKMIND_SEARCH_MCP_HOST_GATEWAY=.*|TKMIND_SEARCH_MCP_HOST_GATEWAY=127.0.0.1|' "${env_file}"
|
||||
fi
|
||||
|
||||
grep -E '^(GOOSED_RUNTIME|TKMIND_API_TARGET|TKMIND_API_TARGETS|GOOSED_MCP_NODE_PATH|MEMIND_GOOSED_HOST_GATEWAY)=' "${env_file}" \
|
||||
| tee "${BACKUP_ROOT}/latest-portal-env-snippet.txt" 2>/dev/null || true
|
||||
}
|
||||
|
||||
wait_for_status() {
|
||||
local port deadline=$((SECONDS + 90))
|
||||
while (( SECONDS < deadline )); do
|
||||
if curl -kfsS --connect-timeout 2 "https://127.0.0.1:${port}/status" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
phase_migrate() {
|
||||
require_103
|
||||
[[ -L "${BACKUP_ROOT}/latest" || -d "${BACKUP_ROOT}" ]] || die "run backup first"
|
||||
install_native_env_files
|
||||
free_disk_headroom
|
||||
stop_docker_pool_keep_standby
|
||||
phase_reconfigure_monitor
|
||||
phase_start_deepseek_proxy
|
||||
install_native_launchd
|
||||
sleep 5
|
||||
for port in "${PORTS[@]}"; do
|
||||
wait_for_status "${port}" || die "native /status failed on ${port}"
|
||||
log "${port}: OK"
|
||||
done
|
||||
update_portal_env
|
||||
log "restarting Portal"
|
||||
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
|
||||
sleep 5
|
||||
curl -fsS "http://127.0.0.1:8081/api/status" >/dev/null || die "Portal /api/status failed"
|
||||
log "syncing LLM provider to all native targets"
|
||||
(
|
||||
cd "${ROOT}"
|
||||
node <<'NODE'
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createDbPool } from './db.mjs';
|
||||
import { createLlmProviderService } from './llm-providers.mjs';
|
||||
|
||||
function loadEnv(file) {
|
||||
if (!fs.existsSync(file)) return;
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t || t.startsWith('#')) continue;
|
||||
const eq = t.indexOf('=');
|
||||
if (eq < 0) continue;
|
||||
const k = t.slice(0, eq).trim();
|
||||
const v = t.slice(eq + 1).trim();
|
||||
if (!process.env[k]) process.env[k] = v;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnv(path.join(process.cwd(), '.env'));
|
||||
const targets = String(process.env.TKMIND_API_TARGETS ?? '')
|
||||
.split(',')
|
||||
.map((v) => v.trim())
|
||||
.filter(Boolean);
|
||||
const pool = createDbPool();
|
||||
const svc = createLlmProviderService(pool, {
|
||||
apiTarget: targets[0] ?? 'https://127.0.0.1:18006',
|
||||
apiTargets: targets,
|
||||
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const synced = await svc.syncSelectedToGoosed();
|
||||
if (!synced.ok) {
|
||||
console.error('provider sync failed:', synced.message);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('provider synced to', synced.targets?.join(', '));
|
||||
await pool.end();
|
||||
NODE
|
||||
) || log "WARN: provider sync failed — verify manually"
|
||||
log "Migration complete. Docker standby: docker start ${DOCKER_STANDBY} (requires stopping native 18006 first)"
|
||||
}
|
||||
|
||||
phase_verify() {
|
||||
require_103
|
||||
for port in "${PORTS[@]}"; do
|
||||
printf '%s: ' "${port}"
|
||||
curl -kfsS --connect-timeout 3 "https://127.0.0.1:${port}/status" && echo || echo FAIL
|
||||
done
|
||||
echo "---"
|
||||
curl -fsS "http://127.0.0.1:8081/api/status" | head -c 400; echo
|
||||
echo "---"
|
||||
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}'
|
||||
launchctl list | grep 'goosed-native' || true
|
||||
}
|
||||
|
||||
phase_rollback() {
|
||||
require_103
|
||||
log "Emergency rollback: stop native pool, start Docker standby on 18006 only"
|
||||
local port label
|
||||
for port in "${PORTS[@]}"; do
|
||||
label="cn.tkmind.goosed-native-${port}"
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
done
|
||||
docker start "${DOCKER_STANDBY}"
|
||||
if [[ -f "${ROOT}/.env.bak-native-${STAMP}" ]]; then
|
||||
cp "${ROOT}/.env.bak-native-${STAMP}" "${ROOT}/.env"
|
||||
elif [[ -f "${BACKUP_ROOT}/latest/portal.env" ]]; then
|
||||
cp "${BACKUP_ROOT}/latest/portal.env" "${ROOT}/.env"
|
||||
fi
|
||||
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
|
||||
curl -kfsS --connect-timeout 5 "https://127.0.0.1:18006/status"
|
||||
log "Rollback to Docker standby on 18006 done. Full pool: cd goosed-prod && docker compose up -d"
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
case "${cmd}" in
|
||||
backup) phase_backup ;;
|
||||
migrate) phase_migrate ;;
|
||||
verify) phase_verify ;;
|
||||
rollback) phase_rollback ;;
|
||||
*)
|
||||
echo "Usage: $0 {backup|migrate|verify|rollback}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
GOOSED_ROOT="${GOOSED_ROOT:-/Users/john/Project/tkmind_go-v141-prod-prep}"
|
||||
RUN_SCRIPT="${GOOSED_RUN_SCRIPT:-${GOOSED_ROOT}/scripts/run-goosed-local.sh}"
|
||||
PORTS="${GOOSED_NATIVE_POOL_PORTS:-18006,18007}"
|
||||
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
|
||||
GUI="gui/$(id -u)"
|
||||
LOG_ROOT="${GOOSED_ROOT}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Install native goosed launchd agents for local multi-instance simulation.
|
||||
|
||||
Usage:
|
||||
bash scripts/install-local-goosed-pool.sh [--ports 18006,18007]
|
||||
|
||||
Environment:
|
||||
GOOSED_ROOT goosed checkout (default: tkmind_go-v141-prod-prep)
|
||||
GOOSED_NATIVE_POOL_PORTS comma-separated host ports
|
||||
|
||||
This script only targets local dev. It refuses prod Portal port 8081 conflicts.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--ports)
|
||||
PORTS="${2:?missing value for --ports}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ! -x "${RUN_SCRIPT}" ]]; then
|
||||
echo "run script not found: ${RUN_SCRIPT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if lsof -nP -iTCP:8081 -sTCP:LISTEN 2>/dev/null | grep -q "cn.tkmind.memind-portal"; then
|
||||
echo "refusing to continue: production Portal appears to own 8081" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${LAUNCHD_DIR}" "${LOG_ROOT}"
|
||||
|
||||
IFS=',' read -r -a port_list <<< "${PORTS}"
|
||||
|
||||
install_port() {
|
||||
local port="$1"
|
||||
local label="com.tkmind.local-goosed-${port}"
|
||||
local plist="${LAUNCHD_DIR}/${label}.plist"
|
||||
local stdout_log="${LOG_ROOT}/goosed.local-${port}.launchd.log"
|
||||
local stderr_log="${LOG_ROOT}/goosed.local-${port}.launchd.err.log"
|
||||
|
||||
cat > "${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>${label}</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/bin/bash</string>
|
||||
<string>${RUN_SCRIPT}</string>
|
||||
<string>${port}</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${GOOSED_ROOT}</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${stdout_log}</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${stderr_log}</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
launchctl enable "${GUI}/${label}"
|
||||
launchctl kickstart -k "${GUI}/${label}"
|
||||
echo "installed ${label} on port ${port}"
|
||||
}
|
||||
|
||||
# Retire legacy single-instance label in favor of per-port labels.
|
||||
launchctl bootout "${GUI}/com.tkmind.local-goosed" 2>/dev/null || true
|
||||
legacy_plist="${LAUNCHD_DIR}/com.tkmind.local-goosed.plist"
|
||||
if [[ -f "${legacy_plist}" ]]; then
|
||||
mv "${legacy_plist}" "${legacy_plist}.bak-$(date +%Y%m%d-%H%M%S)"
|
||||
echo "archived legacy plist: ${legacy_plist}"
|
||||
fi
|
||||
|
||||
for port in "${port_list[@]}"; do
|
||||
port="${port// /}"
|
||||
[[ -n "${port}" ]] || continue
|
||||
install_port "${port}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Verify:"
|
||||
echo " for p in ${PORTS//,/ }; do curl -kfsS https://127.0.0.1:\$p/status; echo \" <- \$p\"; done"
|
||||
echo " node ${ROOT}/scripts/check-local-goosed-pool.mjs"
|
||||
@@ -106,18 +106,8 @@ function cleanupSandboxProcesses() {
|
||||
}
|
||||
|
||||
function goosedLabel(command) {
|
||||
const configured = String(process.env.GOOSED_NATIVE_POOL_PORTS ?? '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
const portPattern = configured.length > 0
|
||||
? configured.map((port) => port.replace(/^:/, '')).join('|')
|
||||
: '1800[6-9]|1801[0-4]';
|
||||
const match = command.match(new RegExp(`\\b(${portPattern})\\b`));
|
||||
const port = match?.[1] ?? null;
|
||||
if (!port) return null;
|
||||
const template = process.env.GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE ?? 'com.tkmind.local-goosed-{port}';
|
||||
return template.replace('{port}', port);
|
||||
const port = command.match(/\b(18006|18007)\b/)?.[1];
|
||||
return port ? `cn.tkmind.goosed-${port}` : null;
|
||||
}
|
||||
|
||||
function monitorGoosed() {
|
||||
|
||||
@@ -54,40 +54,16 @@ function parseMemUsage(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRuntimeMode() {
|
||||
const configured = String(process.env.GOOSED_RUNTIME ?? 'auto').trim().toLowerCase();
|
||||
if (configured === 'native' || configured === 'docker') return configured;
|
||||
try {
|
||||
const line = sh("docker ps --filter 'label=com.docker.compose.project=goosed-prod' --format '{{.Names}}' 2>/dev/null | head -1");
|
||||
return line ? 'docker' : 'native';
|
||||
} catch {
|
||||
return 'native';
|
||||
}
|
||||
}
|
||||
|
||||
function portFromTarget(target) {
|
||||
try {
|
||||
return new URL(target).port || '443';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function targetWorkerIds() {
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
return targets.map((target, index) => {
|
||||
const port = portFromTarget(target);
|
||||
return {
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
port,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
launchdLabel: port ? `com.tkmind.local-goosed-${port}` : null,
|
||||
};
|
||||
});
|
||||
return targets.map((target, index) => ({
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
}));
|
||||
}
|
||||
|
||||
function readDockerStats(container) {
|
||||
@@ -116,51 +92,26 @@ function readContainerFdCount(container) {
|
||||
}
|
||||
}
|
||||
|
||||
function readListenPid(port) {
|
||||
if (!port) return null;
|
||||
try {
|
||||
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
||||
const n = Number(pid);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
}
|
||||
|
||||
function readNativeFdCount(pid) {
|
||||
if (!pid) return 0;
|
||||
try {
|
||||
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
function readNativeProcessStats(pid) {
|
||||
if (!pid) return null;
|
||||
try {
|
||||
const line = sh(`ps -p ${pid} -o %cpu=,rss= 2>/dev/null | tail -1`);
|
||||
const match = line.trim().match(/^([\d.]+)\s+(\d+)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
cpuLoad: Number(match[1]) || 0,
|
||||
rssBytes: (Number(match[2]) || 0) * 1024,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readLaunchdState(label) {
|
||||
if (!label) return null;
|
||||
try {
|
||||
return sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`) || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sampleDockerWorker(worker, fdWarn, now) {
|
||||
const workers = [];
|
||||
for (const worker of targetWorkerIds()) {
|
||||
const stats = readDockerStats(worker.container);
|
||||
const state = readDockerInspect(worker.container);
|
||||
const fdCount = readContainerFdCount(worker.container);
|
||||
@@ -168,9 +119,8 @@ function sampleDockerWorker(worker, fdWarn, now) {
|
||||
const cpuLoad = parsePercent(stats?.CPUPerc);
|
||||
const memoryPressure = parsePercent(stats?.MemPerc);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
return {
|
||||
workers.push({
|
||||
...worker,
|
||||
runtime: 'docker',
|
||||
ok: Boolean(stats && state?.Running),
|
||||
health: state?.Health?.Status ?? state?.Status ?? null,
|
||||
hostPid: Number(state?.Pid ?? 0) || null,
|
||||
@@ -182,60 +132,9 @@ function sampleDockerWorker(worker, fdWarn, now) {
|
||||
memUsedBytes: mem.usedBytes,
|
||||
memLimitBytes: mem.limitBytes,
|
||||
sampledAt: now,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function sampleNativeWorker(worker, fdWarn, now) {
|
||||
const hostPid = readListenPid(worker.port);
|
||||
const fdCount = readNativeFdCount(hostPid);
|
||||
const proc = readNativeProcessStats(hostPid);
|
||||
const launchdState = readLaunchdState(worker.launchdLabel);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
const memUsedBytes = proc?.rssBytes ?? 0;
|
||||
return {
|
||||
...worker,
|
||||
runtime: 'native',
|
||||
ok: Boolean(hostPid && launchdState === 'running'),
|
||||
health: launchdState,
|
||||
hostPid,
|
||||
cpuLoad: proc?.cpuLoad ?? 0,
|
||||
memoryPressure: 0,
|
||||
fdPressure,
|
||||
fdCount,
|
||||
pids: hostPid ? 1 : 0,
|
||||
memUsedBytes,
|
||||
memLimitBytes: 0,
|
||||
sampledAt: now,
|
||||
container: worker.launchdLabel ?? worker.container,
|
||||
};
|
||||
}
|
||||
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
}
|
||||
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const runtimeMode = resolveRuntimeMode();
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
console.error('Env: GOOSED_RUNTIME=auto|native|docker (default auto)');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const workers = targetWorkerIds().map((worker) => (
|
||||
runtimeMode === 'native'
|
||||
? sampleNativeWorker(worker, fdWarn, now)
|
||||
: sampleDockerWorker(worker, fdWarn, now)
|
||||
));
|
||||
|
||||
if (!dryRun) {
|
||||
const client = createClient({ url: redisUrl });
|
||||
client.on('error', (err) => {
|
||||
@@ -264,7 +163,6 @@ console.log(JSON.stringify({
|
||||
ok: workers.every((worker) => worker.ok),
|
||||
action,
|
||||
dryRun,
|
||||
runtimeMode,
|
||||
namespace,
|
||||
redisWrites: !dryRun,
|
||||
workers,
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Local native goosed pool soak: periodic /status + FD sampling across TKMIND_API_TARGETS.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.join(__dirname, '..');
|
||||
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function sh(command) {
|
||||
return execFileSync('/bin/zsh', ['-lc', command], { encoding: 'utf8' }).trim();
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
minutes: Number(process.env.GOOSED_SOAK_MINUTES ?? 30),
|
||||
intervalSec: Number(process.env.GOOSED_SOAK_INTERVAL_SEC ?? 60),
|
||||
};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (arg === '--minutes') {
|
||||
options.minutes = Number(argv[++i]);
|
||||
} else if (arg === '--interval') {
|
||||
options.intervalSec = Number(argv[++i]);
|
||||
} else if (arg === '-h' || arg === '--help') {
|
||||
console.log(`Usage: node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(options.minutes) || options.minutes <= 0) {
|
||||
throw new Error('invalid --minutes');
|
||||
}
|
||||
if (!Number.isFinite(options.intervalSec) || options.intervalSec <= 0) {
|
||||
throw new Error('invalid --interval');
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function resolveTargets() {
|
||||
return (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.flatMap((value) => (
|
||||
value.startsWith('http')
|
||||
? [value.replace(/\/$/, '')]
|
||||
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
|
||||
));
|
||||
}
|
||||
|
||||
function portFromTarget(target) {
|
||||
try {
|
||||
return new URL(target).port || '443';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function nativeLaunchdLabel(port) {
|
||||
return port ? `com.tkmind.local-goosed-${port}` : null;
|
||||
}
|
||||
|
||||
function readListenPid(port) {
|
||||
if (!port) return null;
|
||||
try {
|
||||
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
||||
const n = Number(pid);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readFdCount(pid) {
|
||||
if (!pid) return 0;
|
||||
try {
|
||||
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function readLaunchdState(label) {
|
||||
if (!label) return null;
|
||||
try {
|
||||
const output = sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`);
|
||||
return output || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTarget(target) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const res = await undiciFetch(`${target}/status`, { dispatcher: insecureDispatcher });
|
||||
const body = (await res.text()).trim();
|
||||
return {
|
||||
ok: res.ok && body === 'ok',
|
||||
status: res.status,
|
||||
body,
|
||||
latencyMs: Date.now() - started,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
latencyMs: Date.now() - started,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const targets = resolveTargets();
|
||||
if (targets.length === 0) {
|
||||
console.error('No targets configured.');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const fdRestart = Number(process.env.GOOSED_FD_RESTART ?? 3200);
|
||||
const endAt = Date.now() + options.minutes * 60_000;
|
||||
const summary = {
|
||||
startedAt: new Date().toISOString(),
|
||||
minutes: options.minutes,
|
||||
intervalSec: options.intervalSec,
|
||||
targets,
|
||||
ticks: 0,
|
||||
failures: 0,
|
||||
maxFdByTarget: Object.fromEntries(targets.map((target) => [target, 0])),
|
||||
samples: [],
|
||||
};
|
||||
|
||||
console.log(`[soak] starting ${options.minutes}m interval=${options.intervalSec}s targets=${targets.join(', ')}`);
|
||||
|
||||
while (Date.now() < endAt) {
|
||||
summary.ticks += 1;
|
||||
const tickStarted = Date.now();
|
||||
const tick = {
|
||||
at: new Date().toISOString(),
|
||||
targets: [],
|
||||
ok: true,
|
||||
};
|
||||
|
||||
for (const target of targets) {
|
||||
const port = portFromTarget(target);
|
||||
const label = nativeLaunchdLabel(port);
|
||||
const pid = readListenPid(port);
|
||||
const fdCount = readFdCount(pid);
|
||||
const launchdState = readLaunchdState(label);
|
||||
const probe = await probeTarget(target);
|
||||
|
||||
summary.maxFdByTarget[target] = Math.max(summary.maxFdByTarget[target], fdCount);
|
||||
const entry = {
|
||||
target,
|
||||
port,
|
||||
launchdLabel: label,
|
||||
launchdState,
|
||||
pid,
|
||||
fdCount,
|
||||
fdWarn,
|
||||
fdRestart,
|
||||
fdPressure: fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0,
|
||||
...probe,
|
||||
};
|
||||
tick.targets.push(entry);
|
||||
if (!entry.ok || fdCount >= fdWarn) tick.ok = false;
|
||||
|
||||
const status = entry.ok ? 'ok' : 'FAIL';
|
||||
console.log(
|
||||
`[soak] tick=${summary.ticks} ${target} ${status} pid=${pid ?? '-'} fd=${fdCount} launchd=${launchdState ?? '-'} latency=${entry.latencyMs}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!tick.ok) summary.failures += 1;
|
||||
summary.samples.push(tick);
|
||||
|
||||
const elapsed = Date.now() - tickStarted;
|
||||
const waitMs = Math.max(0, options.intervalSec * 1000 - elapsed);
|
||||
if (Date.now() + waitMs >= endAt) break;
|
||||
await sleep(waitMs);
|
||||
}
|
||||
|
||||
summary.finishedAt = new Date().toISOString();
|
||||
summary.ok = summary.failures === 0;
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: summary.ok,
|
||||
ticks: summary.ticks,
|
||||
failures: summary.failures,
|
||||
maxFdByTarget: summary.maxFdByTarget,
|
||||
fdWarn,
|
||||
fdRestart,
|
||||
}, null, 2));
|
||||
|
||||
insecureDispatcher.close();
|
||||
process.exit(summary.ok ? 0 : 1);
|
||||
Reference in New Issue
Block a user