docs(ops): record 103 goosed native migration and add migrate tooling
Memind CI / Test, build, and release guards (pull_request) Has been cancelled

Document the completed Docker-to-native goosed pool cutover on 103, add the production migrate script with backup/rollback paths, and extend local native pool soak/metrics helpers used as migration gates.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
2026-07-30 20:24:22 +08:00
parent b549a390fe
commit 0dfb2e2b82
9 changed files with 1407 additions and 51 deletions
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
import { Agent, fetch as undiciFetch } from 'undici';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.join(__dirname, '..');
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const idx = trimmed.indexOf('=');
if (idx < 0) continue;
const key = trimmed.slice(0, idx).trim();
const value = trimmed.slice(idx + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
loadEnvFile(path.join(root, '.env'));
const targets = (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
.split(/[,\s]+/)
.map((value) => value.trim())
.filter(Boolean)
.flatMap((value) => (
value.startsWith('http')
? [value]
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
));
if (targets.length === 0) {
console.error('No targets configured. Set TKMIND_API_TARGETS or GOOSED_NATIVE_POOL_PORTS.');
process.exit(2);
}
const results = [];
for (const target of targets) {
const started = Date.now();
try {
const res = await undiciFetch(`${target.replace(/\/$/, '')}/status`, {
dispatcher: insecureDispatcher,
});
const body = (await res.text()).trim();
results.push({
target,
ok: res.ok && body === 'ok',
status: res.status,
body,
ms: Date.now() - started,
});
} catch (err) {
results.push({
target,
ok: false,
error: err instanceof Error ? err.message : String(err),
ms: Date.now() - started,
});
}
}
console.log(JSON.stringify({
ok: results.every((item) => item.ok),
count: results.length,
results,
}, null, 2));
process.exit(results.every((item) => item.ok) ? 0 : 1);
+444
View File
@@ -0,0 +1,444 @@
#!/usr/bin/env bash
# 103 production: Docker goosed pool -> native launchd pool (18006-18014).
# Keeps goosed-prod-1 container stopped as Docker hot standby.
#
# Usage on 103:
# bash scripts/goosed-native-103-migrate.sh backup
# bash scripts/goosed-native-103-migrate.sh migrate
# bash scripts/goosed-native-103-migrate.sh verify
# bash scripts/goosed-native-103-migrate.sh rollback # emergency Docker standby on 18006 only
set -euo pipefail
export PATH="/opt/homebrew/bin:/usr/local/bin:${PATH}"
ROOT="${MEMIND_ROOT:-/Users/john/Project/Memind}"
GOOSED_NATIVE_ROOT="${GOOSED_NATIVE_ROOT:-/Users/john/Project/tkmind_go-native}"
GOOSED_PROD_ROOT="${GOOSED_PROD_ROOT:-/Users/john/Project/goosed-prod}"
PORTS=(18006 18007 18008 18009 18010 18011 18012 18013 18014)
DOCKER_STANDBY="goosed-prod-1"
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
GUI="gui/$(id -u)"
BACKUP_ROOT="${ROOT}/backups/goosed-native-migration"
STAMP="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="${BACKUP_ROOT}/pre-native-${STAMP}"
log() { printf '[goosed-native-103] %s\n' "$*"; }
die() { log "ERROR: $*"; exit 1; }
require_103() {
[[ "$(hostname -s 2>/dev/null || hostname)" == "johndeMac-Studio" || "$(hostname)" == *Studio* ]] || true
[[ -d "${ROOT}" ]] || die "Memind root not found: ${ROOT}"
}
free_disk_headroom() {
log "Rotating oversized logs to free disk headroom before backup"
for f in \
"${HOME}/Library/Logs/memind-agent-run-worker.log" \
"${HOME}/Library/Logs/memind-runtime-heartbeat.log" \
"${HOME}/Library/Logs/memind-frontend-5173.log"; do
if [[ -f "${f}" ]] && [[ "$(wc -c <"${f}" | tr -d ' ')" -gt 50000000 ]]; then
mv "${f}" "${f}.bak-${STAMP}"
: >"${f}"
log "rotated ${f}"
fi
done
df -h "${ROOT}" | tail -1
}
phase_backup() {
require_103
free_disk_headroom
mkdir -p "${BACKUP_DIR}"
log "Backup directory: ${BACKUP_DIR}"
{
echo "stamp=${STAMP}"
echo "hostname=$(hostname)"
date -Iseconds
cat "${ROOT}/.release-manifest.txt" 2>/dev/null || true
grep '^TKMIND_API_TARGETS=' "${ROOT}/.env" || true
grep '^GOOSED_TAG=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null || true
docker ps -a --format '{{.Names}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}' | grep goosed || true
} >"${BACKUP_DIR}/baseline.txt"
cp "${ROOT}/.env" "${BACKUP_DIR}/portal.env"
cp "${GOOSED_PROD_ROOT}/.env" "${BACKUP_DIR}/goosed-prod.env" 2>/dev/null || true
cp "${GOOSED_PROD_ROOT}/docker-compose.prod.yml" "${BACKUP_DIR}/" 2>/dev/null || true
log "pg_dump memind_sessions"
local session_url
session_url="$(grep '^GOOSE_SESSION_DB_URL=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null | cut -d= -f2- | sed 's|@host.docker.internal|@127.0.0.1|' || true)"
if [[ -z "${session_url}" ]]; then
session_url="postgresql://boot:@Abc888888@127.0.0.1:5432/memind_sessions"
fi
pg_dump "${session_url}" | gzip -1 >"${BACKUP_DIR}/memind_sessions.sql.gz"
log "pg_dump mindspace_userdata_prod"
local userdata_url
userdata_url="$(grep '^MINDSPACE_USERDATA_PG_URL=' "${ROOT}/.env" | cut -d= -f2-)"
pg_dump "${userdata_url}" | gzip -1 >"${BACKUP_DIR}/mindspace_userdata_prod.sql.gz"
log "persisted assets tar"
tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
-C "${ROOT}" \
.env MindSpace data users .tailscale public/plaza-covers logs 2>/dev/null \
|| tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
-C "${ROOT}" .env data users logs 2>/dev/null
log "MindSpace storage tar"
tar -czf "${BACKUP_DIR}/mindspace-storage-${STAMP}.tar.gz" \
-C /Users/john/MindSpace data/mindspace 2>/dev/null || true
log "goosed-prod directory tar"
tar -czf "${BACKUP_DIR}/goosed-prod-${STAMP}.tar.gz" \
-C "${GOOSED_PROD_ROOT}" .env docker-compose.prod.yml README.md 2>/dev/null || true
log "tkmind_go-native directory tar"
tar -czf "${BACKUP_DIR}/tkmind_go-native-${STAMP}.tar.gz" \
-C "${GOOSED_NATIVE_ROOT}" \
.env.18006 run-goosed-18006.sh deploy releases goosed 2>/dev/null || true
log "docker inspect goosed containers"
docker ps -aq --filter 'name=goosed-prod' | xargs docker inspect >"${BACKUP_DIR}/docker-goosed-inspect.json" 2>/dev/null || true
log "full Memind source tar (may take a few minutes)"
tar -czf "${BACKUP_DIR}/memind-full-${STAMP}.tar.gz" \
--exclude='./node_modules' \
--exclude='./backups' \
--exclude='./.git' \
-C "${ROOT}" . 2>/dev/null || log "WARN: full tar skipped or partial — check disk space"
ln -sfn "${BACKUP_DIR}" "${BACKUP_ROOT}/latest"
du -sh "${BACKUP_DIR}"/*
df -h "${ROOT}" | tail -1
log "Backup complete: ${BACKUP_DIR}"
}
install_native_env_files() {
local port base="${GOOSED_NATIVE_ROOT}/.env.18006"
[[ -f "${base}" ]] || die "missing ${base}"
for port in "${PORTS[@]}"; do
local env_file="${GOOSED_NATIVE_ROOT}/.env.${port}"
if [[ ! -f "${env_file}" ]]; then
sed "s/^GOOSE_PORT=.*/GOOSE_PORT=${port}/" "${base}" >"${env_file}"
log "created ${env_file}"
fi
done
}
install_native_run_script() {
local script="${GOOSED_NATIVE_ROOT}/run-goosed-native.sh"
cat >"${script}" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
PORT="${1:?missing port argument}"
ROOT="/Users/john/Project/tkmind_go-native"
cd "${ROOT}"
set -a
# shellcheck source=/dev/null
source "${ROOT}/.env.${PORT}"
set +a
exec "${ROOT}/goosed" agent
EOF
chmod +x "${script}"
}
install_native_launchd() {
local port label plist
install_native_run_script
for port in "${PORTS[@]}"; do
label="cn.tkmind.goosed-native-${port}"
plist="${LAUNCHD_DIR}/${label}.plist"
cat >"${plist}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>${label}</string>
<key>ProgramArguments</key>
<array>
<string>${GOOSED_NATIVE_ROOT}/run-goosed-native.sh</string>
<string>${port}</string>
</array>
<key>WorkingDirectory</key>
<string>${GOOSED_NATIVE_ROOT}</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardOutPath</key>
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
<key>StandardErrorPath</key>
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
</dict>
</plist>
EOF
plutil -lint "${plist}" >/dev/null
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
launchctl bootstrap "${GUI}" "${plist}"
launchctl enable "${GUI}/${label}"
launchctl kickstart -k "${GUI}/${label}" 2>/dev/null || launchctl kickstart "${GUI}/${label}"
log "launchd ${label} installed"
done
}
phase_start_deepseek_proxy() {
local plist="${LAUNCHD_DIR}/cn.tkmind.memind-deepseek-no-think.plist"
cat >"${plist}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key><string>cn.tkmind.memind-deepseek-no-think</string>
<key>ProgramArguments</key>
<array>
<string>/opt/homebrew/opt/node@24/bin/node</string>
<string>${ROOT}/deepseek-no-think-proxy.mjs</string>
</array>
<key>EnvironmentVariables</key>
<dict>
<key>MEMIND_DEEPSEEK_PROXY_ENTRYPOINT</key><string>1</string>
<key>MEMIND_DEEPSEEK_DISABLE_THINKING</key><string>1</string>
<key>MEMIND_DEEPSEEK_NO_THINK_PORT</key><string>18036</string>
<key>MEMIND_DEEPSEEK_NO_THINK_HOST</key><string>127.0.0.1</string>
</dict>
<key>WorkingDirectory</key><string>${ROOT}</string>
<key>RunAtLoad</key><true/>
<key>KeepAlive</key><true/>
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
</dict>
</plist>
EOF
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-compat-candidate" 2>/dev/null || true
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-no-think" 2>/dev/null || true
launchctl bootstrap "${GUI}" "${plist}"
curl -fsS "http://127.0.0.1:18036/health" >/dev/null || die "DeepSeek no-think proxy :18036 failed"
log "DeepSeek no-think proxy listening on :18036"
}
phase_reconfigure_monitor() {
local plist="${LAUNCHD_DIR}/cn.tkmind.goosed-monitor.plist"
local monitor_script="${ROOT}/scripts/monitor-goosed-fds.mjs"
[[ -f "${monitor_script}" ]] || die "missing ${monitor_script}"
cp "${plist}" "${plist}.bak-docker-${STAMP}" 2>/dev/null || true
cat >"${plist}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>EnvironmentVariables</key>
<dict>
<key>GOOSED_NATIVE_POOL_PORTS</key>
<string>18006,18007,18008,18009,18010,18011,18012,18013,18014</string>
<key>GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE</key>
<string>cn.tkmind.goosed-native-{port}</string>
<key>GOOSED_FD_RESTART</key>
<string>3200</string>
<key>GOOSED_FD_WARN</key>
<string>180</string>
<key>PATH</key>
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
<key>SANDBOX_MCP_MAX_PER_ROOT</key>
<string>2</string>
<key>SANDBOX_MCP_MAX_TOTAL</key>
<string>80</string>
</dict>
<key>Label</key>
<string>cn.tkmind.goosed-monitor</string>
<key>ProgramArguments</key>
<array>
<string>/opt/homebrew/opt/node@24/bin/node</string>
<string>${monitor_script}</string>
</array>
<key>RunAtLoad</key>
<true/>
<key>StandardErrorPath</key>
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
<key>StandardOutPath</key>
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
<key>StartInterval</key>
<integer>60</integer>
<key>WorkingDirectory</key>
<string>${ROOT}</string>
</dict>
</plist>
EOF
plutil -lint "${plist}" >/dev/null
launchctl bootout "${GUI}/cn.tkmind.goosed-monitor" 2>/dev/null || true
launchctl bootstrap "${GUI}" "${plist}"
log "goosed-monitor switched to native FD monitor"
}
local name
for name in goosed-prod-{1..9}; do
docker update --restart=no "${name}" 2>/dev/null || true
done
for name in goosed-prod-{2..9}; do
if docker ps -q --filter "name=^/${name}$" | grep -q .; then
log "stopping docker ${name}"
docker stop "${name}" >/dev/null
fi
done
if docker ps -q --filter "name=^/${DOCKER_STANDBY}$" | grep -q .; then
log "stopping docker standby ${DOCKER_STANDBY} (container kept, restart=no)"
docker stop "${DOCKER_STANDBY}" >/dev/null
fi
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}' || true
}
update_portal_env() {
local env_file="${ROOT}/.env"
local targets=""
local port
for port in "${PORTS[@]}"; do
targets="${targets}https://127.0.0.1:${port},"
done
targets="${targets%,}"
cp "${env_file}" "${env_file}.bak-native-${STAMP}"
if grep -q '^GOOSED_RUNTIME=' "${env_file}"; then
sed -i '' 's/^GOOSED_RUNTIME=.*/GOOSED_RUNTIME=native/' "${env_file}"
else
printf '\nGOOSED_RUNTIME=native\n' >>"${env_file}"
fi
sed -i '' "s|^TKMIND_API_TARGETS=.*|TKMIND_API_TARGETS=${targets}|" "${env_file}"
sed -i '' 's|^TKMIND_API_TARGET=.*|TKMIND_API_TARGET=https://127.0.0.1:18006|' "${env_file}"
if grep -q '^GOOSED_MCP_NODE_PATH=' "${env_file}"; then
sed -i '' 's|^GOOSED_MCP_NODE_PATH=.*|GOOSED_MCP_NODE_PATH=/opt/homebrew/opt/node@24/bin/node|' "${env_file}"
fi
if grep -q '^MEMIND_GOOSED_HOST_GATEWAY=' "${env_file}"; then
sed -i '' 's|^MEMIND_GOOSED_HOST_GATEWAY=.*|MEMIND_GOOSED_HOST_GATEWAY=127.0.0.1|' "${env_file}"
fi
if grep -q '^TKMIND_SEARCH_MCP_HOST_GATEWAY=' "${env_file}"; then
sed -i '' 's|^TKMIND_SEARCH_MCP_HOST_GATEWAY=.*|TKMIND_SEARCH_MCP_HOST_GATEWAY=127.0.0.1|' "${env_file}"
fi
grep -E '^(GOOSED_RUNTIME|TKMIND_API_TARGET|TKMIND_API_TARGETS|GOOSED_MCP_NODE_PATH|MEMIND_GOOSED_HOST_GATEWAY)=' "${env_file}" \
| tee "${BACKUP_ROOT}/latest-portal-env-snippet.txt" 2>/dev/null || true
}
wait_for_status() {
local port deadline=$((SECONDS + 90))
while (( SECONDS < deadline )); do
if curl -kfsS --connect-timeout 2 "https://127.0.0.1:${port}/status" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
phase_migrate() {
require_103
[[ -L "${BACKUP_ROOT}/latest" || -d "${BACKUP_ROOT}" ]] || die "run backup first"
install_native_env_files
free_disk_headroom
stop_docker_pool_keep_standby
phase_reconfigure_monitor
phase_start_deepseek_proxy
install_native_launchd
sleep 5
for port in "${PORTS[@]}"; do
wait_for_status "${port}" || die "native /status failed on ${port}"
log "${port}: OK"
done
update_portal_env
log "restarting Portal"
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
sleep 5
curl -fsS "http://127.0.0.1:8081/api/status" >/dev/null || die "Portal /api/status failed"
log "syncing LLM provider to all native targets"
(
cd "${ROOT}"
node <<'NODE'
import fs from 'node:fs';
import path from 'node:path';
import { createDbPool } from './db.mjs';
import { createLlmProviderService } from './llm-providers.mjs';
function loadEnv(file) {
if (!fs.existsSync(file)) return;
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
const t = line.trim();
if (!t || t.startsWith('#')) continue;
const eq = t.indexOf('=');
if (eq < 0) continue;
const k = t.slice(0, eq).trim();
const v = t.slice(eq + 1).trim();
if (!process.env[k]) process.env[k] = v;
}
}
loadEnv(path.join(process.cwd(), '.env'));
const targets = String(process.env.TKMIND_API_TARGETS ?? '')
.split(',')
.map((v) => v.trim())
.filter(Boolean);
const pool = createDbPool();
const svc = createLlmProviderService(pool, {
apiTarget: targets[0] ?? 'https://127.0.0.1:18006',
apiTargets: targets,
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
});
const synced = await svc.syncSelectedToGoosed();
if (!synced.ok) {
console.error('provider sync failed:', synced.message);
process.exit(1);
}
console.log('provider synced to', synced.targets?.join(', '));
await pool.end();
NODE
) || log "WARN: provider sync failed — verify manually"
log "Migration complete. Docker standby: docker start ${DOCKER_STANDBY} (requires stopping native 18006 first)"
}
phase_verify() {
require_103
for port in "${PORTS[@]}"; do
printf '%s: ' "${port}"
curl -kfsS --connect-timeout 3 "https://127.0.0.1:${port}/status" && echo || echo FAIL
done
echo "---"
curl -fsS "http://127.0.0.1:8081/api/status" | head -c 400; echo
echo "---"
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}'
launchctl list | grep 'goosed-native' || true
}
phase_rollback() {
require_103
log "Emergency rollback: stop native pool, start Docker standby on 18006 only"
local port label
for port in "${PORTS[@]}"; do
label="cn.tkmind.goosed-native-${port}"
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
done
docker start "${DOCKER_STANDBY}"
if [[ -f "${ROOT}/.env.bak-native-${STAMP}" ]]; then
cp "${ROOT}/.env.bak-native-${STAMP}" "${ROOT}/.env"
elif [[ -f "${BACKUP_ROOT}/latest/portal.env" ]]; then
cp "${BACKUP_ROOT}/latest/portal.env" "${ROOT}/.env"
fi
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
curl -kfsS --connect-timeout 5 "https://127.0.0.1:18006/status"
log "Rollback to Docker standby on 18006 done. Full pool: cd goosed-prod && docker compose up -d"
}
cmd="${1:-}"
case "${cmd}" in
backup) phase_backup ;;
migrate) phase_migrate ;;
verify) phase_verify ;;
rollback) phase_rollback ;;
*)
echo "Usage: $0 {backup|migrate|verify|rollback}" >&2
exit 2
;;
esac
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
GOOSED_ROOT="${GOOSED_ROOT:-/Users/john/Project/tkmind_go-v141-prod-prep}"
RUN_SCRIPT="${GOOSED_RUN_SCRIPT:-${GOOSED_ROOT}/scripts/run-goosed-local.sh}"
PORTS="${GOOSED_NATIVE_POOL_PORTS:-18006,18007}"
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
GUI="gui/$(id -u)"
LOG_ROOT="${GOOSED_ROOT}"
usage() {
cat <<EOF
Install native goosed launchd agents for local multi-instance simulation.
Usage:
bash scripts/install-local-goosed-pool.sh [--ports 18006,18007]
Environment:
GOOSED_ROOT goosed checkout (default: tkmind_go-v141-prod-prep)
GOOSED_NATIVE_POOL_PORTS comma-separated host ports
This script only targets local dev. It refuses prod Portal port 8081 conflicts.
EOF
}
while [[ $# -gt 0 ]]; do
case "$1" in
--ports)
PORTS="${2:?missing value for --ports}"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ ! -x "${RUN_SCRIPT}" ]]; then
echo "run script not found: ${RUN_SCRIPT}" >&2
exit 1
fi
if lsof -nP -iTCP:8081 -sTCP:LISTEN 2>/dev/null | grep -q "cn.tkmind.memind-portal"; then
echo "refusing to continue: production Portal appears to own 8081" >&2
exit 1
fi
mkdir -p "${LAUNCHD_DIR}" "${LOG_ROOT}"
IFS=',' read -r -a port_list <<< "${PORTS}"
install_port() {
local port="$1"
local label="com.tkmind.local-goosed-${port}"
local plist="${LAUNCHD_DIR}/${label}.plist"
local stdout_log="${LOG_ROOT}/goosed.local-${port}.launchd.log"
local stderr_log="${LOG_ROOT}/goosed.local-${port}.launchd.err.log"
cat > "${plist}" <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>${label}</string>
<key>ProgramArguments</key>
<array>
<string>/bin/bash</string>
<string>${RUN_SCRIPT}</string>
<string>${port}</string>
</array>
<key>WorkingDirectory</key>
<string>${GOOSED_ROOT}</string>
<key>RunAtLoad</key>
<true/>
<key>KeepAlive</key>
<true/>
<key>StandardOutPath</key>
<string>${stdout_log}</string>
<key>StandardErrorPath</key>
<string>${stderr_log}</string>
</dict>
</plist>
EOF
plutil -lint "${plist}" >/dev/null
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
launchctl bootstrap "${GUI}" "${plist}"
launchctl enable "${GUI}/${label}"
launchctl kickstart -k "${GUI}/${label}"
echo "installed ${label} on port ${port}"
}
# Retire legacy single-instance label in favor of per-port labels.
launchctl bootout "${GUI}/com.tkmind.local-goosed" 2>/dev/null || true
legacy_plist="${LAUNCHD_DIR}/com.tkmind.local-goosed.plist"
if [[ -f "${legacy_plist}" ]]; then
mv "${legacy_plist}" "${legacy_plist}.bak-$(date +%Y%m%d-%H%M%S)"
echo "archived legacy plist: ${legacy_plist}"
fi
for port in "${port_list[@]}"; do
port="${port// /}"
[[ -n "${port}" ]] || continue
install_port "${port}"
done
echo ""
echo "Verify:"
echo " for p in ${PORTS//,/ }; do curl -kfsS https://127.0.0.1:\$p/status; echo \" <- \$p\"; done"
echo " node ${ROOT}/scripts/check-local-goosed-pool.mjs"
+12 -2
View File
@@ -106,8 +106,18 @@ function cleanupSandboxProcesses() {
}
function goosedLabel(command) {
const port = command.match(/\b(18006|18007)\b/)?.[1];
return port ? `cn.tkmind.goosed-${port}` : null;
const configured = String(process.env.GOOSED_NATIVE_POOL_PORTS ?? '18006,18007')
.split(/[,\s]+/)
.map((value) => value.trim())
.filter(Boolean);
const portPattern = configured.length > 0
? configured.map((port) => port.replace(/^:/, '')).join('|')
: '1800[6-9]|1801[0-4]';
const match = command.match(new RegExp(`\\b(${portPattern})\\b`));
const port = match?.[1] ?? null;
if (!port) return null;
const template = process.env.GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE ?? 'com.tkmind.local-goosed-{port}';
return template.replace('{port}', port);
}
function monitorGoosed() {
+125 -23
View File
@@ -54,16 +54,40 @@ function parseMemUsage(value) {
};
}
function resolveRuntimeMode() {
const configured = String(process.env.GOOSED_RUNTIME ?? 'auto').trim().toLowerCase();
if (configured === 'native' || configured === 'docker') return configured;
try {
const line = sh("docker ps --filter 'label=com.docker.compose.project=goosed-prod' --format '{{.Names}}' 2>/dev/null | head -1");
return line ? 'docker' : 'native';
} catch {
return 'native';
}
}
function portFromTarget(target) {
try {
return new URL(target).port || '443';
} catch {
return null;
}
}
function targetWorkerIds() {
const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
.split(',')
.map((value) => value.trim())
.filter(Boolean);
return targets.map((target, index) => ({
id: `goosed-${index + 1}`,
target,
container: `goosed-prod-${index + 1}`,
}));
return targets.map((target, index) => {
const port = portFromTarget(target);
return {
id: `goosed-${index + 1}`,
target,
port,
container: `goosed-prod-${index + 1}`,
launchdLabel: port ? `com.tkmind.local-goosed-${port}` : null,
};
});
}
function readDockerStats(container) {
@@ -92,26 +116,51 @@ function readContainerFdCount(container) {
}
}
function workerKey(namespace, id, field) {
return [namespace, 'worker', id, field].join(':');
function readListenPid(port) {
if (!port) return null;
try {
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
const n = Number(pid);
return Number.isFinite(n) && n > 0 ? n : null;
} catch {
return null;
}
}
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
const action = process.argv[2] || 'sample';
const dryRun = process.argv.includes('--dry-run') || action === 'status';
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
const now = Date.now();
if (!['sample', 'status'].includes(action)) {
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
process.exit(2);
function readNativeFdCount(pid) {
if (!pid) return 0;
try {
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
} catch {
return 0;
}
}
const workers = [];
for (const worker of targetWorkerIds()) {
function readNativeProcessStats(pid) {
if (!pid) return null;
try {
const line = sh(`ps -p ${pid} -o %cpu=,rss= 2>/dev/null | tail -1`);
const match = line.trim().match(/^([\d.]+)\s+(\d+)$/);
if (!match) return null;
return {
cpuLoad: Number(match[1]) || 0,
rssBytes: (Number(match[2]) || 0) * 1024,
};
} catch {
return null;
}
}
function readLaunchdState(label) {
if (!label) return null;
try {
return sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`) || null;
} catch {
return null;
}
}
function sampleDockerWorker(worker, fdWarn, now) {
const stats = readDockerStats(worker.container);
const state = readDockerInspect(worker.container);
const fdCount = readContainerFdCount(worker.container);
@@ -119,8 +168,9 @@ for (const worker of targetWorkerIds()) {
const cpuLoad = parsePercent(stats?.CPUPerc);
const memoryPressure = parsePercent(stats?.MemPerc);
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
workers.push({
return {
...worker,
runtime: 'docker',
ok: Boolean(stats && state?.Running),
health: state?.Health?.Status ?? state?.Status ?? null,
hostPid: Number(state?.Pid ?? 0) || null,
@@ -132,9 +182,60 @@ for (const worker of targetWorkerIds()) {
memUsedBytes: mem.usedBytes,
memLimitBytes: mem.limitBytes,
sampledAt: now,
});
};
}
function sampleNativeWorker(worker, fdWarn, now) {
const hostPid = readListenPid(worker.port);
const fdCount = readNativeFdCount(hostPid);
const proc = readNativeProcessStats(hostPid);
const launchdState = readLaunchdState(worker.launchdLabel);
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
const memUsedBytes = proc?.rssBytes ?? 0;
return {
...worker,
runtime: 'native',
ok: Boolean(hostPid && launchdState === 'running'),
health: launchdState,
hostPid,
cpuLoad: proc?.cpuLoad ?? 0,
memoryPressure: 0,
fdPressure,
fdCount,
pids: hostPid ? 1 : 0,
memUsedBytes,
memLimitBytes: 0,
sampledAt: now,
container: worker.launchdLabel ?? worker.container,
};
}
function workerKey(namespace, id, field) {
return [namespace, 'worker', id, field].join(':');
}
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
const runtimeMode = resolveRuntimeMode();
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
const action = process.argv[2] || 'sample';
const dryRun = process.argv.includes('--dry-run') || action === 'status';
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
const now = Date.now();
if (!['sample', 'status'].includes(action)) {
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
console.error('Env: GOOSED_RUNTIME=auto|native|docker (default auto)');
process.exit(2);
}
const workers = targetWorkerIds().map((worker) => (
runtimeMode === 'native'
? sampleNativeWorker(worker, fdWarn, now)
: sampleDockerWorker(worker, fdWarn, now)
));
if (!dryRun) {
const client = createClient({ url: redisUrl });
client.on('error', (err) => {
@@ -163,6 +264,7 @@ console.log(JSON.stringify({
ok: workers.every((worker) => worker.ok),
action,
dryRun,
runtimeMode,
namespace,
redisWrites: !dryRun,
workers,
+224
View File
@@ -0,0 +1,224 @@
#!/usr/bin/env node
/**
* Local native goosed pool soak: periodic /status + FD sampling across TKMIND_API_TARGETS.
*
* Usage:
* node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]
*/
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { Agent, fetch as undiciFetch } from 'undici';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const root = path.join(__dirname, '..');
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
function loadEnvFile(filePath) {
if (!fs.existsSync(filePath)) return;
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
const trimmed = line.trim();
if (!trimmed || trimmed.startsWith('#')) continue;
const idx = trimmed.indexOf('=');
if (idx < 0) continue;
const key = trimmed.slice(0, idx).trim();
const value = trimmed.slice(idx + 1).trim();
if (!process.env[key]) process.env[key] = value;
}
}
function sh(command) {
return execFileSync('/bin/zsh', ['-lc', command], { encoding: 'utf8' }).trim();
}
function parseArgs(argv) {
const options = {
minutes: Number(process.env.GOOSED_SOAK_MINUTES ?? 30),
intervalSec: Number(process.env.GOOSED_SOAK_INTERVAL_SEC ?? 60),
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === '--minutes') {
options.minutes = Number(argv[++i]);
} else if (arg === '--interval') {
options.intervalSec = Number(argv[++i]);
} else if (arg === '-h' || arg === '--help') {
console.log(`Usage: node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]`);
process.exit(0);
}
}
if (!Number.isFinite(options.minutes) || options.minutes <= 0) {
throw new Error('invalid --minutes');
}
if (!Number.isFinite(options.intervalSec) || options.intervalSec <= 0) {
throw new Error('invalid --interval');
}
return options;
}
function resolveTargets() {
return (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
.split(/[,\s]+/)
.map((value) => value.trim())
.filter(Boolean)
.flatMap((value) => (
value.startsWith('http')
? [value.replace(/\/$/, '')]
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
));
}
function portFromTarget(target) {
try {
return new URL(target).port || '443';
} catch {
return null;
}
}
function nativeLaunchdLabel(port) {
return port ? `com.tkmind.local-goosed-${port}` : null;
}
function readListenPid(port) {
if (!port) return null;
try {
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
const n = Number(pid);
return Number.isFinite(n) && n > 0 ? n : null;
} catch {
return null;
}
}
function readFdCount(pid) {
if (!pid) return 0;
try {
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
} catch {
return 0;
}
}
function readLaunchdState(label) {
if (!label) return null;
try {
const output = sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`);
return output || null;
} catch {
return null;
}
}
async function probeTarget(target) {
const started = Date.now();
try {
const res = await undiciFetch(`${target}/status`, { dispatcher: insecureDispatcher });
const body = (await res.text()).trim();
return {
ok: res.ok && body === 'ok',
status: res.status,
body,
latencyMs: Date.now() - started,
};
} catch (err) {
return {
ok: false,
latencyMs: Date.now() - started,
error: err instanceof Error ? err.message : String(err),
};
}
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
loadEnvFile(path.join(root, '.env'));
const options = parseArgs(process.argv.slice(2));
const targets = resolveTargets();
if (targets.length === 0) {
console.error('No targets configured.');
process.exit(2);
}
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
const fdRestart = Number(process.env.GOOSED_FD_RESTART ?? 3200);
const endAt = Date.now() + options.minutes * 60_000;
const summary = {
startedAt: new Date().toISOString(),
minutes: options.minutes,
intervalSec: options.intervalSec,
targets,
ticks: 0,
failures: 0,
maxFdByTarget: Object.fromEntries(targets.map((target) => [target, 0])),
samples: [],
};
console.log(`[soak] starting ${options.minutes}m interval=${options.intervalSec}s targets=${targets.join(', ')}`);
while (Date.now() < endAt) {
summary.ticks += 1;
const tickStarted = Date.now();
const tick = {
at: new Date().toISOString(),
targets: [],
ok: true,
};
for (const target of targets) {
const port = portFromTarget(target);
const label = nativeLaunchdLabel(port);
const pid = readListenPid(port);
const fdCount = readFdCount(pid);
const launchdState = readLaunchdState(label);
const probe = await probeTarget(target);
summary.maxFdByTarget[target] = Math.max(summary.maxFdByTarget[target], fdCount);
const entry = {
target,
port,
launchdLabel: label,
launchdState,
pid,
fdCount,
fdWarn,
fdRestart,
fdPressure: fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0,
...probe,
};
tick.targets.push(entry);
if (!entry.ok || fdCount >= fdWarn) tick.ok = false;
const status = entry.ok ? 'ok' : 'FAIL';
console.log(
`[soak] tick=${summary.ticks} ${target} ${status} pid=${pid ?? '-'} fd=${fdCount} launchd=${launchdState ?? '-'} latency=${entry.latencyMs}ms`,
);
}
if (!tick.ok) summary.failures += 1;
summary.samples.push(tick);
const elapsed = Date.now() - tickStarted;
const waitMs = Math.max(0, options.intervalSec * 1000 - elapsed);
if (Date.now() + waitMs >= endAt) break;
await sleep(waitMs);
}
summary.finishedAt = new Date().toISOString();
summary.ok = summary.failures === 0;
console.log(JSON.stringify({
ok: summary.ok,
ticks: summary.ticks,
failures: summary.failures,
maxFdByTarget: summary.maxFdByTarget,
fdWarn,
fdRestart,
}, null, 2));
insecureDispatcher.close();
process.exit(summary.ok ? 0 : 1);