fix(goosed-ops): raise goosed nofile limit and add 103 pool hardening script
launchd leaves goosed at a 256 soft nofile limit, causing EMFILE on busy sessions. The run script now raises ulimit -n; the hardening script applies this and the harness shim to the live pool with backup, rolling restart, health checks and --rollback. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
john
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# Harden the 103 native goosed v1.49 pool (:18006-:18014):
|
||||
# 1. raise the per-process open-file soft limit in run-goosed-native.sh (EMFILE on extension load)
|
||||
# 2. point CODEX_HARNESS_MYSQL at the Portal-shipped mysql2 shim (harness remember failures)
|
||||
# Then rolling-restart one instance at a time with a health check.
|
||||
#
|
||||
# Self-contained so it can run from the dev Mac without a checkout on 103:
|
||||
# ssh john@114.85.107.50 'bash -s' -- --dry-run < scripts/goose-v149-pool-hardening-103.sh
|
||||
# ssh john@114.85.107.50 'bash -s' -- --yes < scripts/goose-v149-pool-hardening-103.sh
|
||||
# ssh john@114.85.107.50 'bash -s' -- --rollback < scripts/goose-v149-pool-hardening-103.sh
|
||||
set -euo pipefail
|
||||
|
||||
NATIVE_ROOT="${GOOSE_NATIVE_ROOT:-/Users/john/Project/tkmind_go-native}"
|
||||
PORTAL_ROOT="${MEMIND_PORTAL_ROOT:-/Users/john/Project/Memind}"
|
||||
HARNESS_SHIM="${PORTAL_ROOT}/scripts/goosed-harness-mysql2-cli.mjs"
|
||||
NOFILE="${GOOSED_NOFILE:-65536}"
|
||||
PORTS="${GOOSED_POOL_PORTS:-18006 18007 18008 18009 18010 18011 18012 18013 18014}"
|
||||
RUN_SCRIPT="${NATIVE_ROOT}/run-goosed-native.sh"
|
||||
GUI="gui/$(id -u)"
|
||||
HEALTH_WAIT_SECS="${GOOSED_HEALTH_WAIT_SECS:-45}"
|
||||
|
||||
say() { printf '[goose-v149-hardening] %s\n' "$*"; }
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: goose-v149-pool-hardening-103.sh --dry-run | --yes | --rollback
|
||||
--dry-run show planned changes, touch nothing
|
||||
--yes back up, apply, rolling restart
|
||||
--rollback restore run script + .env.* from the latest pre-hardening backup, rolling restart
|
||||
EOF
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
local port="$1" i
|
||||
for ((i = 0; i < HEALTH_WAIT_SECS; i++)); do
|
||||
if curl -kfsS --connect-timeout 2 "https://127.0.0.1:${port}/status" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
rolling_restart() {
|
||||
local port
|
||||
for port in ${PORTS}; do
|
||||
say "restart :${port}"
|
||||
launchctl kickstart -k "${GUI}/cn.tkmind.goosed-native-${port}"
|
||||
if ! wait_healthy "${port}"; then
|
||||
say "FAIL :${port} not healthy after ${HEALTH_WAIT_SECS}s; stopping rollout" >&2
|
||||
return 1
|
||||
fi
|
||||
say "ok :${port}"
|
||||
done
|
||||
}
|
||||
|
||||
write_run_script() {
|
||||
cat > "$1" <<RUNEOF
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
export PATH="/opt/homebrew/opt/node@24/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"
|
||||
PORT="\${1:?missing port argument}"
|
||||
ROOT="${NATIVE_ROOT}"
|
||||
cd "\${ROOT}"
|
||||
ulimit -n "\${GOOSED_NOFILE:-${NOFILE}}" 2>/dev/null || ulimit -n 10240 2>/dev/null || true
|
||||
echo "GOOSED_NOFILE=\$(ulimit -n) port=\${PORT}" >&2
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "\${ROOT}/.env.\${PORT}"
|
||||
set +a
|
||||
GOOSE_BIN="\${GOOSE_BIN:-\${ROOT}/goosed}"
|
||||
exec env "CUSTOM_DEEPSEEK-CP_API_KEY=\${DEEPSEEK_API_KEY:-}" \\
|
||||
"\${GOOSE_BIN}" serve \\
|
||||
--host "\${GOOSE_HOST:-127.0.0.1}" \\
|
||||
--port "\${PORT}" \\
|
||||
--tls
|
||||
RUNEOF
|
||||
chmod +x "$1"
|
||||
}
|
||||
|
||||
set_env_value() {
|
||||
local file="$1" key="$2" value="$3" tmp
|
||||
tmp="$(mktemp)"
|
||||
awk -v k="${key}" -v v="${value}" '
|
||||
BEGIN { done = 0 }
|
||||
index($0, k "=") == 1 { if (!done) { print k "=" v; done = 1 } ; next }
|
||||
{ print }
|
||||
END { if (!done) print k "=" v }
|
||||
' "${file}" > "${tmp}"
|
||||
cat "${tmp}" > "${file}"
|
||||
rm -f "${tmp}"
|
||||
}
|
||||
|
||||
harness_shim_usable() {
|
||||
[[ -f "${HARNESS_SHIM}" ]] || return 1
|
||||
local env_file="${NATIVE_ROOT}/.env.$(echo "${PORTS}" | awk '{print $1}')"
|
||||
(
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "${env_file}"
|
||||
set +a
|
||||
printf 'SELECT 1' | node "${HARNESS_SHIM}" -h "${CODEX_HARNESS_DB_HOST:-127.0.0.1}" \
|
||||
-u "${CODEX_HARNESS_DB_USER:-boot}" -N "${CODEX_HARNESS_DB_NAME:-codex_harness}" >/dev/null
|
||||
)
|
||||
}
|
||||
|
||||
rollback() {
|
||||
local latest env
|
||||
latest="$(ls -dt "${NATIVE_ROOT}"/backups/pre-hardening-* 2>/dev/null | head -1 || true)"
|
||||
[[ -n "${latest}" ]] || { echo "No pre-hardening backup under ${NATIVE_ROOT}/backups/" >&2; exit 1; }
|
||||
say "rollback from ${latest}"
|
||||
cp "${latest}/run-goosed-native.sh" "${RUN_SCRIPT}"
|
||||
chmod +x "${RUN_SCRIPT}"
|
||||
for env in "${latest}"/.env.*; do
|
||||
[[ -f "${env}" ]] && cp "${env}" "${NATIVE_ROOT}/$(basename "${env}")"
|
||||
done
|
||||
rolling_restart
|
||||
say "rollback complete"
|
||||
}
|
||||
|
||||
MODE="${1:-}"
|
||||
case "${MODE}" in
|
||||
--rollback) rollback; exit 0 ;;
|
||||
--dry-run|--yes) ;;
|
||||
*) usage >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
APPLY_HARNESS=1
|
||||
if harness_shim_usable; then
|
||||
say "harness shim ok: ${HARNESS_SHIM}"
|
||||
else
|
||||
APPLY_HARNESS=0
|
||||
say "WARN harness shim missing or cannot query DB; skipping CODEX_HARNESS_MYSQL change" >&2
|
||||
fi
|
||||
|
||||
say "plan: ulimit -n ${NOFILE} in ${RUN_SCRIPT}"
|
||||
[[ "${APPLY_HARNESS}" == 1 ]] && say "plan: CODEX_HARNESS_MYSQL=${HARNESS_SHIM} in .env.{${PORTS// /,}}"
|
||||
say "plan: rolling restart ${PORTS}"
|
||||
|
||||
if [[ "${MODE}" == "--dry-run" ]]; then
|
||||
say "current soft limit for launchd jobs: $(launchctl limit maxfiles | awk '{print $2}')"
|
||||
grep -h '^CODEX_HARNESS_MYSQL=' "${NATIVE_ROOT}"/.env.18006 || true
|
||||
say "dry run: no changes made"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BACKUP_DIR="${NATIVE_ROOT}/backups/pre-hardening-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
cp -a "${RUN_SCRIPT}" "${BACKUP_DIR}/"
|
||||
for port in ${PORTS}; do
|
||||
cp -a "${NATIVE_ROOT}/.env.${port}" "${BACKUP_DIR}/"
|
||||
done
|
||||
say "backup ${BACKUP_DIR}"
|
||||
|
||||
write_run_script "${RUN_SCRIPT}"
|
||||
if [[ "${APPLY_HARNESS}" == 1 ]]; then
|
||||
for port in ${PORTS}; do
|
||||
set_env_value "${NATIVE_ROOT}/.env.${port}" CODEX_HARNESS_MYSQL "${HARNESS_SHIM}"
|
||||
done
|
||||
fi
|
||||
|
||||
if ! rolling_restart; then
|
||||
say "rollout halted; restore with: --rollback (backup ${BACKUP_DIR})" >&2
|
||||
exit 1
|
||||
fi
|
||||
say "hardening complete; check 'GOOSED_NOFILE=' lines in ~/Library/Logs/goosed-native-*.log"
|
||||
@@ -96,6 +96,8 @@ export PATH="/opt/homebrew/opt/node@24/bin:/opt/homebrew/bin:/usr/local/bin:/usr
|
||||
PORT="${1:?missing port argument}"
|
||||
ROOT="/Users/john/Project/tkmind_go-native"
|
||||
cd "${ROOT}"
|
||||
ulimit -n "${GOOSED_NOFILE:-65536}" 2>/dev/null || ulimit -n 10240 2>/dev/null || true
|
||||
echo "GOOSED_NOFILE=$(ulimit -n) port=${PORT}" >&2
|
||||
set -a
|
||||
# shellcheck source=/dev/null
|
||||
source "${ROOT}/.env.${PORT}"
|
||||
|
||||
Reference in New Issue
Block a user