docs(ops): record 103 goosed native migration and add migrate tooling
Memind CI / Test, build, and release guards (pull_request) Has been cancelled
Memind CI / Test, build, and release guards (pull_request) Has been cancelled
Document the completed Docker-to-native goosed pool cutover on 103, add the production migrate script with backup/rollback paths, and extend local native pool soak/metrics helpers used as migration gates. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.join(__dirname, '..');
|
||||
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.flatMap((value) => (
|
||||
value.startsWith('http')
|
||||
? [value]
|
||||
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
|
||||
));
|
||||
|
||||
if (targets.length === 0) {
|
||||
console.error('No targets configured. Set TKMIND_API_TARGETS or GOOSED_NATIVE_POOL_PORTS.');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const target of targets) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const res = await undiciFetch(`${target.replace(/\/$/, '')}/status`, {
|
||||
dispatcher: insecureDispatcher,
|
||||
});
|
||||
const body = (await res.text()).trim();
|
||||
results.push({
|
||||
target,
|
||||
ok: res.ok && body === 'ok',
|
||||
status: res.status,
|
||||
body,
|
||||
ms: Date.now() - started,
|
||||
});
|
||||
} catch (err) {
|
||||
results.push({
|
||||
target,
|
||||
ok: false,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
ms: Date.now() - started,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: results.every((item) => item.ok),
|
||||
count: results.length,
|
||||
results,
|
||||
}, null, 2));
|
||||
|
||||
process.exit(results.every((item) => item.ok) ? 0 : 1);
|
||||
@@ -0,0 +1,444 @@
|
||||
#!/usr/bin/env bash
|
||||
# 103 production: Docker goosed pool -> native launchd pool (18006-18014).
|
||||
# Keeps goosed-prod-1 container stopped as Docker hot standby.
|
||||
#
|
||||
# Usage on 103:
|
||||
# bash scripts/goosed-native-103-migrate.sh backup
|
||||
# bash scripts/goosed-native-103-migrate.sh migrate
|
||||
# bash scripts/goosed-native-103-migrate.sh verify
|
||||
# bash scripts/goosed-native-103-migrate.sh rollback # emergency Docker standby on 18006 only
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="/opt/homebrew/bin:/usr/local/bin:${PATH}"
|
||||
|
||||
ROOT="${MEMIND_ROOT:-/Users/john/Project/Memind}"
|
||||
GOOSED_NATIVE_ROOT="${GOOSED_NATIVE_ROOT:-/Users/john/Project/tkmind_go-native}"
|
||||
GOOSED_PROD_ROOT="${GOOSED_PROD_ROOT:-/Users/john/Project/goosed-prod}"
|
||||
PORTS=(18006 18007 18008 18009 18010 18011 18012 18013 18014)
|
||||
DOCKER_STANDBY="goosed-prod-1"
|
||||
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
|
||||
GUI="gui/$(id -u)"
|
||||
BACKUP_ROOT="${ROOT}/backups/goosed-native-migration"
|
||||
STAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="${BACKUP_ROOT}/pre-native-${STAMP}"
|
||||
|
||||
log() { printf '[goosed-native-103] %s\n' "$*"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
|
||||
require_103() {
|
||||
[[ "$(hostname -s 2>/dev/null || hostname)" == "johndeMac-Studio" || "$(hostname)" == *Studio* ]] || true
|
||||
[[ -d "${ROOT}" ]] || die "Memind root not found: ${ROOT}"
|
||||
}
|
||||
|
||||
free_disk_headroom() {
|
||||
log "Rotating oversized logs to free disk headroom before backup"
|
||||
for f in \
|
||||
"${HOME}/Library/Logs/memind-agent-run-worker.log" \
|
||||
"${HOME}/Library/Logs/memind-runtime-heartbeat.log" \
|
||||
"${HOME}/Library/Logs/memind-frontend-5173.log"; do
|
||||
if [[ -f "${f}" ]] && [[ "$(wc -c <"${f}" | tr -d ' ')" -gt 50000000 ]]; then
|
||||
mv "${f}" "${f}.bak-${STAMP}"
|
||||
: >"${f}"
|
||||
log "rotated ${f}"
|
||||
fi
|
||||
done
|
||||
df -h "${ROOT}" | tail -1
|
||||
}
|
||||
|
||||
phase_backup() {
|
||||
require_103
|
||||
free_disk_headroom
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
log "Backup directory: ${BACKUP_DIR}"
|
||||
|
||||
{
|
||||
echo "stamp=${STAMP}"
|
||||
echo "hostname=$(hostname)"
|
||||
date -Iseconds
|
||||
cat "${ROOT}/.release-manifest.txt" 2>/dev/null || true
|
||||
grep '^TKMIND_API_TARGETS=' "${ROOT}/.env" || true
|
||||
grep '^GOOSED_TAG=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null || true
|
||||
docker ps -a --format '{{.Names}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}' | grep goosed || true
|
||||
} >"${BACKUP_DIR}/baseline.txt"
|
||||
|
||||
cp "${ROOT}/.env" "${BACKUP_DIR}/portal.env"
|
||||
cp "${GOOSED_PROD_ROOT}/.env" "${BACKUP_DIR}/goosed-prod.env" 2>/dev/null || true
|
||||
cp "${GOOSED_PROD_ROOT}/docker-compose.prod.yml" "${BACKUP_DIR}/" 2>/dev/null || true
|
||||
|
||||
log "pg_dump memind_sessions"
|
||||
local session_url
|
||||
session_url="$(grep '^GOOSE_SESSION_DB_URL=' "${GOOSED_PROD_ROOT}/.env" 2>/dev/null | cut -d= -f2- | sed 's|@host.docker.internal|@127.0.0.1|' || true)"
|
||||
if [[ -z "${session_url}" ]]; then
|
||||
session_url="postgresql://boot:@Abc888888@127.0.0.1:5432/memind_sessions"
|
||||
fi
|
||||
pg_dump "${session_url}" | gzip -1 >"${BACKUP_DIR}/memind_sessions.sql.gz"
|
||||
|
||||
log "pg_dump mindspace_userdata_prod"
|
||||
local userdata_url
|
||||
userdata_url="$(grep '^MINDSPACE_USERDATA_PG_URL=' "${ROOT}/.env" | cut -d= -f2-)"
|
||||
pg_dump "${userdata_url}" | gzip -1 >"${BACKUP_DIR}/mindspace_userdata_prod.sql.gz"
|
||||
|
||||
log "persisted assets tar"
|
||||
tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
|
||||
-C "${ROOT}" \
|
||||
.env MindSpace data users .tailscale public/plaza-covers logs 2>/dev/null \
|
||||
|| tar -czf "${BACKUP_DIR}/memind-persisted-${STAMP}.tar.gz" \
|
||||
-C "${ROOT}" .env data users logs 2>/dev/null
|
||||
|
||||
log "MindSpace storage tar"
|
||||
tar -czf "${BACKUP_DIR}/mindspace-storage-${STAMP}.tar.gz" \
|
||||
-C /Users/john/MindSpace data/mindspace 2>/dev/null || true
|
||||
|
||||
log "goosed-prod directory tar"
|
||||
tar -czf "${BACKUP_DIR}/goosed-prod-${STAMP}.tar.gz" \
|
||||
-C "${GOOSED_PROD_ROOT}" .env docker-compose.prod.yml README.md 2>/dev/null || true
|
||||
|
||||
log "tkmind_go-native directory tar"
|
||||
tar -czf "${BACKUP_DIR}/tkmind_go-native-${STAMP}.tar.gz" \
|
||||
-C "${GOOSED_NATIVE_ROOT}" \
|
||||
.env.18006 run-goosed-18006.sh deploy releases goosed 2>/dev/null || true
|
||||
|
||||
log "docker inspect goosed containers"
|
||||
docker ps -aq --filter 'name=goosed-prod' | xargs docker inspect >"${BACKUP_DIR}/docker-goosed-inspect.json" 2>/dev/null || true
|
||||
|
||||
log "full Memind source tar (may take a few minutes)"
|
||||
tar -czf "${BACKUP_DIR}/memind-full-${STAMP}.tar.gz" \
|
||||
--exclude='./node_modules' \
|
||||
--exclude='./backups' \
|
||||
--exclude='./.git' \
|
||||
-C "${ROOT}" . 2>/dev/null || log "WARN: full tar skipped or partial — check disk space"
|
||||
|
||||
ln -sfn "${BACKUP_DIR}" "${BACKUP_ROOT}/latest"
|
||||
du -sh "${BACKUP_DIR}"/*
|
||||
df -h "${ROOT}" | tail -1
|
||||
log "Backup complete: ${BACKUP_DIR}"
|
||||
}
|
||||
|
||||
install_native_env_files() {
|
||||
local port base="${GOOSED_NATIVE_ROOT}/.env.18006"
|
||||
[[ -f "${base}" ]] || die "missing ${base}"
|
||||
for port in "${PORTS[@]}"; do
|
||||
local env_file="${GOOSED_NATIVE_ROOT}/.env.${port}"
|
||||
if [[ ! -f "${env_file}" ]]; then
|
||||
sed "s/^GOOSE_PORT=.*/GOOSE_PORT=${port}/" "${base}" >"${env_file}"
|
||||
log "created ${env_file}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
install_native_run_script() {
|
||||
local script="${GOOSED_NATIVE_ROOT}/run-goosed-native.sh"
|
||||
cat >"${script}" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
PORT="${1:?missing port argument}"
|
||||
ROOT="/Users/john/Project/tkmind_go-native"
|
||||
cd "${ROOT}"
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "${ROOT}/.env.${PORT}"
|
||||
set +a
|
||||
exec "${ROOT}/goosed" agent
|
||||
EOF
|
||||
chmod +x "${script}"
|
||||
}
|
||||
|
||||
install_native_launchd() {
|
||||
local port label plist
|
||||
install_native_run_script
|
||||
for port in "${PORTS[@]}"; do
|
||||
label="cn.tkmind.goosed-native-${port}"
|
||||
plist="${LAUNCHD_DIR}/${label}.plist"
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>${label}</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>${GOOSED_NATIVE_ROOT}/run-goosed-native.sh</string>
|
||||
<string>${port}</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${GOOSED_NATIVE_ROOT}</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-native-${port}.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
launchctl enable "${GUI}/${label}"
|
||||
launchctl kickstart -k "${GUI}/${label}" 2>/dev/null || launchctl kickstart "${GUI}/${label}"
|
||||
log "launchd ${label} installed"
|
||||
done
|
||||
}
|
||||
|
||||
phase_start_deepseek_proxy() {
|
||||
local plist="${LAUNCHD_DIR}/cn.tkmind.memind-deepseek-no-think.plist"
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key><string>cn.tkmind.memind-deepseek-no-think</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/opt/homebrew/opt/node@24/bin/node</string>
|
||||
<string>${ROOT}/deepseek-no-think-proxy.mjs</string>
|
||||
</array>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>MEMIND_DEEPSEEK_PROXY_ENTRYPOINT</key><string>1</string>
|
||||
<key>MEMIND_DEEPSEEK_DISABLE_THINKING</key><string>1</string>
|
||||
<key>MEMIND_DEEPSEEK_NO_THINK_PORT</key><string>18036</string>
|
||||
<key>MEMIND_DEEPSEEK_NO_THINK_HOST</key><string>127.0.0.1</string>
|
||||
</dict>
|
||||
<key>WorkingDirectory</key><string>${ROOT}</string>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
<key>StandardOutPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
|
||||
<key>StandardErrorPath</key><string>${HOME}/Library/Logs/memind-deepseek-no-think.log</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-compat-candidate" 2>/dev/null || true
|
||||
launchctl bootout "${GUI}/cn.tkmind.memind-deepseek-no-think" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
curl -fsS "http://127.0.0.1:18036/health" >/dev/null || die "DeepSeek no-think proxy :18036 failed"
|
||||
log "DeepSeek no-think proxy listening on :18036"
|
||||
}
|
||||
|
||||
phase_reconfigure_monitor() {
|
||||
local plist="${LAUNCHD_DIR}/cn.tkmind.goosed-monitor.plist"
|
||||
local monitor_script="${ROOT}/scripts/monitor-goosed-fds.mjs"
|
||||
[[ -f "${monitor_script}" ]] || die "missing ${monitor_script}"
|
||||
cp "${plist}" "${plist}.bak-docker-${STAMP}" 2>/dev/null || true
|
||||
cat >"${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>GOOSED_NATIVE_POOL_PORTS</key>
|
||||
<string>18006,18007,18008,18009,18010,18011,18012,18013,18014</string>
|
||||
<key>GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE</key>
|
||||
<string>cn.tkmind.goosed-native-{port}</string>
|
||||
<key>GOOSED_FD_RESTART</key>
|
||||
<string>3200</string>
|
||||
<key>GOOSED_FD_WARN</key>
|
||||
<string>180</string>
|
||||
<key>PATH</key>
|
||||
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||
<key>SANDBOX_MCP_MAX_PER_ROOT</key>
|
||||
<string>2</string>
|
||||
<key>SANDBOX_MCP_MAX_TOTAL</key>
|
||||
<string>80</string>
|
||||
</dict>
|
||||
<key>Label</key>
|
||||
<string>cn.tkmind.goosed-monitor</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/opt/homebrew/opt/node@24/bin/node</string>
|
||||
<string>${monitor_script}</string>
|
||||
</array>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${HOME}/Library/Logs/goosed-monitor.log</string>
|
||||
<key>StartInterval</key>
|
||||
<integer>60</integer>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${ROOT}</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/cn.tkmind.goosed-monitor" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
log "goosed-monitor switched to native FD monitor"
|
||||
}
|
||||
|
||||
local name
|
||||
for name in goosed-prod-{1..9}; do
|
||||
docker update --restart=no "${name}" 2>/dev/null || true
|
||||
done
|
||||
for name in goosed-prod-{2..9}; do
|
||||
if docker ps -q --filter "name=^/${name}$" | grep -q .; then
|
||||
log "stopping docker ${name}"
|
||||
docker stop "${name}" >/dev/null
|
||||
fi
|
||||
done
|
||||
if docker ps -q --filter "name=^/${DOCKER_STANDBY}$" | grep -q .; then
|
||||
log "stopping docker standby ${DOCKER_STANDBY} (container kept, restart=no)"
|
||||
docker stop "${DOCKER_STANDBY}" >/dev/null
|
||||
fi
|
||||
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}' || true
|
||||
}
|
||||
|
||||
update_portal_env() {
|
||||
local env_file="${ROOT}/.env"
|
||||
local targets=""
|
||||
local port
|
||||
for port in "${PORTS[@]}"; do
|
||||
targets="${targets}https://127.0.0.1:${port},"
|
||||
done
|
||||
targets="${targets%,}"
|
||||
|
||||
cp "${env_file}" "${env_file}.bak-native-${STAMP}"
|
||||
|
||||
if grep -q '^GOOSED_RUNTIME=' "${env_file}"; then
|
||||
sed -i '' 's/^GOOSED_RUNTIME=.*/GOOSED_RUNTIME=native/' "${env_file}"
|
||||
else
|
||||
printf '\nGOOSED_RUNTIME=native\n' >>"${env_file}"
|
||||
fi
|
||||
|
||||
sed -i '' "s|^TKMIND_API_TARGETS=.*|TKMIND_API_TARGETS=${targets}|" "${env_file}"
|
||||
sed -i '' 's|^TKMIND_API_TARGET=.*|TKMIND_API_TARGET=https://127.0.0.1:18006|' "${env_file}"
|
||||
|
||||
if grep -q '^GOOSED_MCP_NODE_PATH=' "${env_file}"; then
|
||||
sed -i '' 's|^GOOSED_MCP_NODE_PATH=.*|GOOSED_MCP_NODE_PATH=/opt/homebrew/opt/node@24/bin/node|' "${env_file}"
|
||||
fi
|
||||
if grep -q '^MEMIND_GOOSED_HOST_GATEWAY=' "${env_file}"; then
|
||||
sed -i '' 's|^MEMIND_GOOSED_HOST_GATEWAY=.*|MEMIND_GOOSED_HOST_GATEWAY=127.0.0.1|' "${env_file}"
|
||||
fi
|
||||
if grep -q '^TKMIND_SEARCH_MCP_HOST_GATEWAY=' "${env_file}"; then
|
||||
sed -i '' 's|^TKMIND_SEARCH_MCP_HOST_GATEWAY=.*|TKMIND_SEARCH_MCP_HOST_GATEWAY=127.0.0.1|' "${env_file}"
|
||||
fi
|
||||
|
||||
grep -E '^(GOOSED_RUNTIME|TKMIND_API_TARGET|TKMIND_API_TARGETS|GOOSED_MCP_NODE_PATH|MEMIND_GOOSED_HOST_GATEWAY)=' "${env_file}" \
|
||||
| tee "${BACKUP_ROOT}/latest-portal-env-snippet.txt" 2>/dev/null || true
|
||||
}
|
||||
|
||||
wait_for_status() {
|
||||
local port deadline=$((SECONDS + 90))
|
||||
while (( SECONDS < deadline )); do
|
||||
if curl -kfsS --connect-timeout 2 "https://127.0.0.1:${port}/status" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
phase_migrate() {
|
||||
require_103
|
||||
[[ -L "${BACKUP_ROOT}/latest" || -d "${BACKUP_ROOT}" ]] || die "run backup first"
|
||||
install_native_env_files
|
||||
free_disk_headroom
|
||||
stop_docker_pool_keep_standby
|
||||
phase_reconfigure_monitor
|
||||
phase_start_deepseek_proxy
|
||||
install_native_launchd
|
||||
sleep 5
|
||||
for port in "${PORTS[@]}"; do
|
||||
wait_for_status "${port}" || die "native /status failed on ${port}"
|
||||
log "${port}: OK"
|
||||
done
|
||||
update_portal_env
|
||||
log "restarting Portal"
|
||||
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
|
||||
sleep 5
|
||||
curl -fsS "http://127.0.0.1:8081/api/status" >/dev/null || die "Portal /api/status failed"
|
||||
log "syncing LLM provider to all native targets"
|
||||
(
|
||||
cd "${ROOT}"
|
||||
node <<'NODE'
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createDbPool } from './db.mjs';
|
||||
import { createLlmProviderService } from './llm-providers.mjs';
|
||||
|
||||
function loadEnv(file) {
|
||||
if (!fs.existsSync(file)) return;
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
const t = line.trim();
|
||||
if (!t || t.startsWith('#')) continue;
|
||||
const eq = t.indexOf('=');
|
||||
if (eq < 0) continue;
|
||||
const k = t.slice(0, eq).trim();
|
||||
const v = t.slice(eq + 1).trim();
|
||||
if (!process.env[k]) process.env[k] = v;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnv(path.join(process.cwd(), '.env'));
|
||||
const targets = String(process.env.TKMIND_API_TARGETS ?? '')
|
||||
.split(',')
|
||||
.map((v) => v.trim())
|
||||
.filter(Boolean);
|
||||
const pool = createDbPool();
|
||||
const svc = createLlmProviderService(pool, {
|
||||
apiTarget: targets[0] ?? 'https://127.0.0.1:18006',
|
||||
apiTargets: targets,
|
||||
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const synced = await svc.syncSelectedToGoosed();
|
||||
if (!synced.ok) {
|
||||
console.error('provider sync failed:', synced.message);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('provider synced to', synced.targets?.join(', '));
|
||||
await pool.end();
|
||||
NODE
|
||||
) || log "WARN: provider sync failed — verify manually"
|
||||
log "Migration complete. Docker standby: docker start ${DOCKER_STANDBY} (requires stopping native 18006 first)"
|
||||
}
|
||||
|
||||
phase_verify() {
|
||||
require_103
|
||||
for port in "${PORTS[@]}"; do
|
||||
printf '%s: ' "${port}"
|
||||
curl -kfsS --connect-timeout 3 "https://127.0.0.1:${port}/status" && echo || echo FAIL
|
||||
done
|
||||
echo "---"
|
||||
curl -fsS "http://127.0.0.1:8081/api/status" | head -c 400; echo
|
||||
echo "---"
|
||||
docker ps -a --filter 'name=goosed-prod' --format '{{.Names}}\t{{.Status}}'
|
||||
launchctl list | grep 'goosed-native' || true
|
||||
}
|
||||
|
||||
phase_rollback() {
|
||||
require_103
|
||||
log "Emergency rollback: stop native pool, start Docker standby on 18006 only"
|
||||
local port label
|
||||
for port in "${PORTS[@]}"; do
|
||||
label="cn.tkmind.goosed-native-${port}"
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
done
|
||||
docker start "${DOCKER_STANDBY}"
|
||||
if [[ -f "${ROOT}/.env.bak-native-${STAMP}" ]]; then
|
||||
cp "${ROOT}/.env.bak-native-${STAMP}" "${ROOT}/.env"
|
||||
elif [[ -f "${BACKUP_ROOT}/latest/portal.env" ]]; then
|
||||
cp "${BACKUP_ROOT}/latest/portal.env" "${ROOT}/.env"
|
||||
fi
|
||||
launchctl kickstart -k "${GUI}/cn.tkmind.memind-portal"
|
||||
curl -kfsS --connect-timeout 5 "https://127.0.0.1:18006/status"
|
||||
log "Rollback to Docker standby on 18006 done. Full pool: cd goosed-prod && docker compose up -d"
|
||||
}
|
||||
|
||||
cmd="${1:-}"
|
||||
case "${cmd}" in
|
||||
backup) phase_backup ;;
|
||||
migrate) phase_migrate ;;
|
||||
verify) phase_verify ;;
|
||||
rollback) phase_rollback ;;
|
||||
*)
|
||||
echo "Usage: $0 {backup|migrate|verify|rollback}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
GOOSED_ROOT="${GOOSED_ROOT:-/Users/john/Project/tkmind_go-v141-prod-prep}"
|
||||
RUN_SCRIPT="${GOOSED_RUN_SCRIPT:-${GOOSED_ROOT}/scripts/run-goosed-local.sh}"
|
||||
PORTS="${GOOSED_NATIVE_POOL_PORTS:-18006,18007}"
|
||||
LAUNCHD_DIR="${HOME}/Library/LaunchAgents"
|
||||
GUI="gui/$(id -u)"
|
||||
LOG_ROOT="${GOOSED_ROOT}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Install native goosed launchd agents for local multi-instance simulation.
|
||||
|
||||
Usage:
|
||||
bash scripts/install-local-goosed-pool.sh [--ports 18006,18007]
|
||||
|
||||
Environment:
|
||||
GOOSED_ROOT goosed checkout (default: tkmind_go-v141-prod-prep)
|
||||
GOOSED_NATIVE_POOL_PORTS comma-separated host ports
|
||||
|
||||
This script only targets local dev. It refuses prod Portal port 8081 conflicts.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--ports)
|
||||
PORTS="${2:?missing value for --ports}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ! -x "${RUN_SCRIPT}" ]]; then
|
||||
echo "run script not found: ${RUN_SCRIPT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if lsof -nP -iTCP:8081 -sTCP:LISTEN 2>/dev/null | grep -q "cn.tkmind.memind-portal"; then
|
||||
echo "refusing to continue: production Portal appears to own 8081" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "${LAUNCHD_DIR}" "${LOG_ROOT}"
|
||||
|
||||
IFS=',' read -r -a port_list <<< "${PORTS}"
|
||||
|
||||
install_port() {
|
||||
local port="$1"
|
||||
local label="com.tkmind.local-goosed-${port}"
|
||||
local plist="${LAUNCHD_DIR}/${label}.plist"
|
||||
local stdout_log="${LOG_ROOT}/goosed.local-${port}.launchd.log"
|
||||
local stderr_log="${LOG_ROOT}/goosed.local-${port}.launchd.err.log"
|
||||
|
||||
cat > "${plist}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
||||
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>${label}</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>/bin/bash</string>
|
||||
<string>${RUN_SCRIPT}</string>
|
||||
<string>${port}</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>${GOOSED_ROOT}</string>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>KeepAlive</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>${stdout_log}</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>${stderr_log}</string>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
plutil -lint "${plist}" >/dev/null
|
||||
launchctl bootout "${GUI}/${label}" 2>/dev/null || true
|
||||
launchctl bootstrap "${GUI}" "${plist}"
|
||||
launchctl enable "${GUI}/${label}"
|
||||
launchctl kickstart -k "${GUI}/${label}"
|
||||
echo "installed ${label} on port ${port}"
|
||||
}
|
||||
|
||||
# Retire legacy single-instance label in favor of per-port labels.
|
||||
launchctl bootout "${GUI}/com.tkmind.local-goosed" 2>/dev/null || true
|
||||
legacy_plist="${LAUNCHD_DIR}/com.tkmind.local-goosed.plist"
|
||||
if [[ -f "${legacy_plist}" ]]; then
|
||||
mv "${legacy_plist}" "${legacy_plist}.bak-$(date +%Y%m%d-%H%M%S)"
|
||||
echo "archived legacy plist: ${legacy_plist}"
|
||||
fi
|
||||
|
||||
for port in "${port_list[@]}"; do
|
||||
port="${port// /}"
|
||||
[[ -n "${port}" ]] || continue
|
||||
install_port "${port}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Verify:"
|
||||
echo " for p in ${PORTS//,/ }; do curl -kfsS https://127.0.0.1:\$p/status; echo \" <- \$p\"; done"
|
||||
echo " node ${ROOT}/scripts/check-local-goosed-pool.mjs"
|
||||
@@ -106,8 +106,18 @@ function cleanupSandboxProcesses() {
|
||||
}
|
||||
|
||||
function goosedLabel(command) {
|
||||
const port = command.match(/\b(18006|18007)\b/)?.[1];
|
||||
return port ? `cn.tkmind.goosed-${port}` : null;
|
||||
const configured = String(process.env.GOOSED_NATIVE_POOL_PORTS ?? '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
const portPattern = configured.length > 0
|
||||
? configured.map((port) => port.replace(/^:/, '')).join('|')
|
||||
: '1800[6-9]|1801[0-4]';
|
||||
const match = command.match(new RegExp(`\\b(${portPattern})\\b`));
|
||||
const port = match?.[1] ?? null;
|
||||
if (!port) return null;
|
||||
const template = process.env.GOOSED_NATIVE_LAUNCHD_LABEL_TEMPLATE ?? 'com.tkmind.local-goosed-{port}';
|
||||
return template.replace('{port}', port);
|
||||
}
|
||||
|
||||
function monitorGoosed() {
|
||||
|
||||
@@ -54,16 +54,40 @@ function parseMemUsage(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRuntimeMode() {
|
||||
const configured = String(process.env.GOOSED_RUNTIME ?? 'auto').trim().toLowerCase();
|
||||
if (configured === 'native' || configured === 'docker') return configured;
|
||||
try {
|
||||
const line = sh("docker ps --filter 'label=com.docker.compose.project=goosed-prod' --format '{{.Names}}' 2>/dev/null | head -1");
|
||||
return line ? 'docker' : 'native';
|
||||
} catch {
|
||||
return 'native';
|
||||
}
|
||||
}
|
||||
|
||||
function portFromTarget(target) {
|
||||
try {
|
||||
return new URL(target).port || '443';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function targetWorkerIds() {
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
return targets.map((target, index) => ({
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
}));
|
||||
return targets.map((target, index) => {
|
||||
const port = portFromTarget(target);
|
||||
return {
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
port,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
launchdLabel: port ? `com.tkmind.local-goosed-${port}` : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function readDockerStats(container) {
|
||||
@@ -92,26 +116,51 @@ function readContainerFdCount(container) {
|
||||
}
|
||||
}
|
||||
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
function readListenPid(port) {
|
||||
if (!port) return null;
|
||||
try {
|
||||
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
||||
const n = Number(pid);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
process.exit(2);
|
||||
function readNativeFdCount(pid) {
|
||||
if (!pid) return 0;
|
||||
try {
|
||||
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
const workers = [];
|
||||
for (const worker of targetWorkerIds()) {
|
||||
function readNativeProcessStats(pid) {
|
||||
if (!pid) return null;
|
||||
try {
|
||||
const line = sh(`ps -p ${pid} -o %cpu=,rss= 2>/dev/null | tail -1`);
|
||||
const match = line.trim().match(/^([\d.]+)\s+(\d+)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
cpuLoad: Number(match[1]) || 0,
|
||||
rssBytes: (Number(match[2]) || 0) * 1024,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readLaunchdState(label) {
|
||||
if (!label) return null;
|
||||
try {
|
||||
return sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`) || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sampleDockerWorker(worker, fdWarn, now) {
|
||||
const stats = readDockerStats(worker.container);
|
||||
const state = readDockerInspect(worker.container);
|
||||
const fdCount = readContainerFdCount(worker.container);
|
||||
@@ -119,8 +168,9 @@ for (const worker of targetWorkerIds()) {
|
||||
const cpuLoad = parsePercent(stats?.CPUPerc);
|
||||
const memoryPressure = parsePercent(stats?.MemPerc);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
workers.push({
|
||||
return {
|
||||
...worker,
|
||||
runtime: 'docker',
|
||||
ok: Boolean(stats && state?.Running),
|
||||
health: state?.Health?.Status ?? state?.Status ?? null,
|
||||
hostPid: Number(state?.Pid ?? 0) || null,
|
||||
@@ -132,9 +182,60 @@ for (const worker of targetWorkerIds()) {
|
||||
memUsedBytes: mem.usedBytes,
|
||||
memLimitBytes: mem.limitBytes,
|
||||
sampledAt: now,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function sampleNativeWorker(worker, fdWarn, now) {
|
||||
const hostPid = readListenPid(worker.port);
|
||||
const fdCount = readNativeFdCount(hostPid);
|
||||
const proc = readNativeProcessStats(hostPid);
|
||||
const launchdState = readLaunchdState(worker.launchdLabel);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
const memUsedBytes = proc?.rssBytes ?? 0;
|
||||
return {
|
||||
...worker,
|
||||
runtime: 'native',
|
||||
ok: Boolean(hostPid && launchdState === 'running'),
|
||||
health: launchdState,
|
||||
hostPid,
|
||||
cpuLoad: proc?.cpuLoad ?? 0,
|
||||
memoryPressure: 0,
|
||||
fdPressure,
|
||||
fdCount,
|
||||
pids: hostPid ? 1 : 0,
|
||||
memUsedBytes,
|
||||
memLimitBytes: 0,
|
||||
sampledAt: now,
|
||||
container: worker.launchdLabel ?? worker.container,
|
||||
};
|
||||
}
|
||||
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
}
|
||||
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const runtimeMode = resolveRuntimeMode();
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
console.error('Env: GOOSED_RUNTIME=auto|native|docker (default auto)');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const workers = targetWorkerIds().map((worker) => (
|
||||
runtimeMode === 'native'
|
||||
? sampleNativeWorker(worker, fdWarn, now)
|
||||
: sampleDockerWorker(worker, fdWarn, now)
|
||||
));
|
||||
|
||||
if (!dryRun) {
|
||||
const client = createClient({ url: redisUrl });
|
||||
client.on('error', (err) => {
|
||||
@@ -163,6 +264,7 @@ console.log(JSON.stringify({
|
||||
ok: workers.every((worker) => worker.ok),
|
||||
action,
|
||||
dryRun,
|
||||
runtimeMode,
|
||||
namespace,
|
||||
redisWrites: !dryRun,
|
||||
workers,
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Local native goosed pool soak: periodic /status + FD sampling across TKMIND_API_TARGETS.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]
|
||||
*/
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.join(__dirname, '..');
|
||||
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function sh(command) {
|
||||
return execFileSync('/bin/zsh', ['-lc', command], { encoding: 'utf8' }).trim();
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
minutes: Number(process.env.GOOSED_SOAK_MINUTES ?? 30),
|
||||
intervalSec: Number(process.env.GOOSED_SOAK_INTERVAL_SEC ?? 60),
|
||||
};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (arg === '--minutes') {
|
||||
options.minutes = Number(argv[++i]);
|
||||
} else if (arg === '--interval') {
|
||||
options.intervalSec = Number(argv[++i]);
|
||||
} else if (arg === '-h' || arg === '--help') {
|
||||
console.log(`Usage: node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]`);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
if (!Number.isFinite(options.minutes) || options.minutes <= 0) {
|
||||
throw new Error('invalid --minutes');
|
||||
}
|
||||
if (!Number.isFinite(options.intervalSec) || options.intervalSec <= 0) {
|
||||
throw new Error('invalid --interval');
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function resolveTargets() {
|
||||
return (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
|
||||
.split(/[,\s]+/)
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.flatMap((value) => (
|
||||
value.startsWith('http')
|
||||
? [value.replace(/\/$/, '')]
|
||||
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
|
||||
));
|
||||
}
|
||||
|
||||
function portFromTarget(target) {
|
||||
try {
|
||||
return new URL(target).port || '443';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function nativeLaunchdLabel(port) {
|
||||
return port ? `com.tkmind.local-goosed-${port}` : null;
|
||||
}
|
||||
|
||||
function readListenPid(port) {
|
||||
if (!port) return null;
|
||||
try {
|
||||
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
||||
const n = Number(pid);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readFdCount(pid) {
|
||||
if (!pid) return 0;
|
||||
try {
|
||||
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function readLaunchdState(label) {
|
||||
if (!label) return null;
|
||||
try {
|
||||
const output = sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`);
|
||||
return output || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTarget(target) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
const res = await undiciFetch(`${target}/status`, { dispatcher: insecureDispatcher });
|
||||
const body = (await res.text()).trim();
|
||||
return {
|
||||
ok: res.ok && body === 'ok',
|
||||
status: res.status,
|
||||
body,
|
||||
latencyMs: Date.now() - started,
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
latencyMs: Date.now() - started,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(root, '.env'));
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const targets = resolveTargets();
|
||||
if (targets.length === 0) {
|
||||
console.error('No targets configured.');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const fdRestart = Number(process.env.GOOSED_FD_RESTART ?? 3200);
|
||||
const endAt = Date.now() + options.minutes * 60_000;
|
||||
const summary = {
|
||||
startedAt: new Date().toISOString(),
|
||||
minutes: options.minutes,
|
||||
intervalSec: options.intervalSec,
|
||||
targets,
|
||||
ticks: 0,
|
||||
failures: 0,
|
||||
maxFdByTarget: Object.fromEntries(targets.map((target) => [target, 0])),
|
||||
samples: [],
|
||||
};
|
||||
|
||||
console.log(`[soak] starting ${options.minutes}m interval=${options.intervalSec}s targets=${targets.join(', ')}`);
|
||||
|
||||
while (Date.now() < endAt) {
|
||||
summary.ticks += 1;
|
||||
const tickStarted = Date.now();
|
||||
const tick = {
|
||||
at: new Date().toISOString(),
|
||||
targets: [],
|
||||
ok: true,
|
||||
};
|
||||
|
||||
for (const target of targets) {
|
||||
const port = portFromTarget(target);
|
||||
const label = nativeLaunchdLabel(port);
|
||||
const pid = readListenPid(port);
|
||||
const fdCount = readFdCount(pid);
|
||||
const launchdState = readLaunchdState(label);
|
||||
const probe = await probeTarget(target);
|
||||
|
||||
summary.maxFdByTarget[target] = Math.max(summary.maxFdByTarget[target], fdCount);
|
||||
const entry = {
|
||||
target,
|
||||
port,
|
||||
launchdLabel: label,
|
||||
launchdState,
|
||||
pid,
|
||||
fdCount,
|
||||
fdWarn,
|
||||
fdRestart,
|
||||
fdPressure: fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0,
|
||||
...probe,
|
||||
};
|
||||
tick.targets.push(entry);
|
||||
if (!entry.ok || fdCount >= fdWarn) tick.ok = false;
|
||||
|
||||
const status = entry.ok ? 'ok' : 'FAIL';
|
||||
console.log(
|
||||
`[soak] tick=${summary.ticks} ${target} ${status} pid=${pid ?? '-'} fd=${fdCount} launchd=${launchdState ?? '-'} latency=${entry.latencyMs}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!tick.ok) summary.failures += 1;
|
||||
summary.samples.push(tick);
|
||||
|
||||
const elapsed = Date.now() - tickStarted;
|
||||
const waitMs = Math.max(0, options.intervalSec * 1000 - elapsed);
|
||||
if (Date.now() + waitMs >= endAt) break;
|
||||
await sleep(waitMs);
|
||||
}
|
||||
|
||||
summary.finishedAt = new Date().toISOString();
|
||||
summary.ok = summary.failures === 0;
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: summary.ok,
|
||||
ticks: summary.ticks,
|
||||
failures: summary.failures,
|
||||
maxFdByTarget: summary.maxFdByTarget,
|
||||
fdWarn,
|
||||
fdRestart,
|
||||
}, null, 2));
|
||||
|
||||
insecureDispatcher.close();
|
||||
process.exit(summary.ok ? 0 : 1);
|
||||
Reference in New Issue
Block a user