memind/scripts/goose-v149-pool-hardening-103.sh

#!/usr/bin/env bash
# Harden the 103 native goosed v1.49 pool (:18006-:18014):
#   1. raise the per-process open-file soft limit in run-goosed-native.sh (EMFILE on extension load)
#   2. point CODEX_HARNESS_MYSQL at the Portal-shipped mysql2 shim (harness remember failures)
# Then rolling-restart one instance at a time with a health check.
#
# Self-contained so it can run from the dev Mac without a checkout on 103:
#   ssh john@114.85.107.50 'bash -s' -- --dry-run < scripts/goose-v149-pool-hardening-103.sh
#   ssh john@114.85.107.50 'bash -s' -- --yes     < scripts/goose-v149-pool-hardening-103.sh
#   ssh john@114.85.107.50 'bash -s' -- --rollback < scripts/goose-v149-pool-hardening-103.sh
set -euo pipefail

NATIVE_ROOT="${GOOSE_NATIVE_ROOT:-/Users/john/Project/tkmind_go-native}"
PORTAL_ROOT="${MEMIND_PORTAL_ROOT:-/Users/john/Project/Memind}"
HARNESS_SHIM="${PORTAL_ROOT}/scripts/goosed-harness-mysql2-cli.mjs"
NOFILE="${GOOSED_NOFILE:-65536}"
PORTS="${GOOSED_POOL_PORTS:-18006 18007 18008 18009 18010 18011 18012 18013 18014}"
RUN_SCRIPT="${NATIVE_ROOT}/run-goosed-native.sh"
GUI="gui/$(id -u)"
HEALTH_WAIT_SECS="${GOOSED_HEALTH_WAIT_SECS:-45}"

say() { printf '[goose-v149-hardening] %s\n' "$*"; }

usage() {
  cat <<EOF
Usage: goose-v149-pool-hardening-103.sh --dry-run | --yes | --rollback
  --dry-run   show planned changes, touch nothing
  --yes       back up, apply, rolling restart
  --rollback  restore run script + .env.* from the latest pre-hardening backup, rolling restart
EOF
}

wait_healthy() {
  local port="$1" i
  for ((i = 0; i < HEALTH_WAIT_SECS; i++)); do
    if curl -kfsS --connect-timeout 2 "https://127.0.0.1:${port}/status" >/dev/null 2>&1; then
      return 0
    fi
    sleep 1
  done
  return 1
}

rolling_restart() {
  local port
  for port in ${PORTS}; do
    say "restart :${port}"
    launchctl kickstart -k "${GUI}/cn.tkmind.goosed-native-${port}"
    if ! wait_healthy "${port}"; then
      say "FAIL :${port} not healthy after ${HEALTH_WAIT_SECS}s; stopping rollout" >&2
      return 1
    fi
    say "ok :${port}"
  done
}

write_run_script() {
  cat > "$1" <<RUNEOF
#!/usr/bin/env bash
set -euo pipefail
export PATH="/opt/homebrew/opt/node@24/bin:/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"
PORT="\${1:?missing port argument}"
ROOT="${NATIVE_ROOT}"
cd "\${ROOT}"
ulimit -n "\${GOOSED_NOFILE:-${NOFILE}}" 2>/dev/null || ulimit -n 10240 2>/dev/null || true
echo "GOOSED_NOFILE=\$(ulimit -n) port=\${PORT}" >&2
set -a
# shellcheck source=/dev/null
source "\${ROOT}/.env.\${PORT}"
set +a
GOOSE_BIN="\${GOOSE_BIN:-\${ROOT}/goosed}"
exec env "CUSTOM_DEEPSEEK-CP_API_KEY=\${DEEPSEEK_API_KEY:-}" \\
  "\${GOOSE_BIN}" serve \\
  --host "\${GOOSE_HOST:-127.0.0.1}" \\
  --port "\${PORT}" \\
  --tls
RUNEOF
  chmod +x "$1"
}

set_env_value() {
  local file="$1" key="$2" value="$3" tmp
  tmp="$(mktemp)"
  awk -v k="${key}" -v v="${value}" '
    BEGIN { done = 0 }
    index($0, k "=") == 1 { if (!done) { print k "=" v; done = 1 } ; next }
    { print }
    END { if (!done) print k "=" v }
  ' "${file}" > "${tmp}"
  cat "${tmp}" > "${file}"
  rm -f "${tmp}"
}

harness_shim_usable() {
  [[ -f "${HARNESS_SHIM}" ]] || return 1
  local env_file="${NATIVE_ROOT}/.env.$(echo "${PORTS}" | awk '{print $1}')"
  (
    set -a
    # shellcheck source=/dev/null
    source "${env_file}"
    set +a
    printf 'SELECT 1' | node "${HARNESS_SHIM}" -h "${CODEX_HARNESS_DB_HOST:-127.0.0.1}" \
      -u "${CODEX_HARNESS_DB_USER:-boot}" -N "${CODEX_HARNESS_DB_NAME:-codex_harness}" >/dev/null
  )
}

rollback() {
  local latest env
  latest="$(ls -dt "${NATIVE_ROOT}"/backups/pre-hardening-* 2>/dev/null | head -1 || true)"
  [[ -n "${latest}" ]] || { echo "No pre-hardening backup under ${NATIVE_ROOT}/backups/" >&2; exit 1; }
  say "rollback from ${latest}"
  cp "${latest}/run-goosed-native.sh" "${RUN_SCRIPT}"
  chmod +x "${RUN_SCRIPT}"
  for env in "${latest}"/.env.*; do
    [[ -f "${env}" ]] && cp "${env}" "${NATIVE_ROOT}/$(basename "${env}")"
  done
  rolling_restart
  say "rollback complete"
}

MODE="${1:-}"
case "${MODE}" in
  --rollback) rollback; exit 0 ;;
  --dry-run|--yes) ;;
  *) usage >&2; exit 2 ;;
esac

APPLY_HARNESS=1
if harness_shim_usable; then
  say "harness shim ok: ${HARNESS_SHIM}"
else
  APPLY_HARNESS=0
  say "WARN harness shim missing or cannot query DB; skipping CODEX_HARNESS_MYSQL change" >&2
fi

say "plan: ulimit -n ${NOFILE} in ${RUN_SCRIPT}"
[[ "${APPLY_HARNESS}" == 1 ]] && say "plan: CODEX_HARNESS_MYSQL=${HARNESS_SHIM} in .env.{${PORTS// /,}}"
say "plan: rolling restart ${PORTS}"

if [[ "${MODE}" == "--dry-run" ]]; then
  say "current soft limit for launchd jobs: $(launchctl limit maxfiles | awk '{print $2}')"
  grep -h '^CODEX_HARNESS_MYSQL=' "${NATIVE_ROOT}"/.env.18006 || true
  say "dry run: no changes made"
  exit 0
fi

BACKUP_DIR="${NATIVE_ROOT}/backups/pre-hardening-$(date +%Y%m%d-%H%M%S)"
mkdir -p "${BACKUP_DIR}"
cp -a "${RUN_SCRIPT}" "${BACKUP_DIR}/"
for port in ${PORTS}; do
  cp -a "${NATIVE_ROOT}/.env.${port}" "${BACKUP_DIR}/"
done
say "backup ${BACKUP_DIR}"

write_run_script "${RUN_SCRIPT}"
if [[ "${APPLY_HARNESS}" == 1 ]]; then
  for port in ${PORTS}; do
    set_env_value "${NATIVE_ROOT}/.env.${port}" CODEX_HARNESS_MYSQL "${HARNESS_SHIM}"
  done
fi

if ! rolling_restart; then
  say "rollout halted; restore with: --rollback (backup ${BACKUP_DIR})" >&2
  exit 1
fi
say "hardening complete; check 'GOOSED_NOFILE=' lines in ~/Library/Logs/goosed-native-*.log"