memind/scripts/runtime-worker-metrics.mjs

#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
import { execFileSync } from 'node:child_process';
import { createClient } from 'redis';

function loadEnvFile(filePath) {
  if (!fs.existsSync(filePath)) return;
  for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith('#')) continue;
    const idx = trimmed.indexOf('=');
    if (idx < 0) continue;
    const key = trimmed.slice(0, idx).trim();
    const value = trimmed.slice(idx + 1).trim();
    if (!process.env[key]) process.env[key] = value;
  }
}

function sh(command) {
  return execFileSync('/bin/zsh', ['-lc', command], { encoding: 'utf8' }).trim();
}

function parsePercent(value) {
  const n = Number(String(value ?? '').replace('%', '').trim());
  return Number.isFinite(n) ? n : 0;
}

function parseBytes(value) {
  const match = String(value ?? '').trim().match(/^([\d.]+)\s*([kmgt]?i?b)?$/i);
  if (!match) return 0;
  const n = Number(match[1]);
  if (!Number.isFinite(n)) return 0;
  const unit = (match[2] || 'b').toLowerCase();
  const scale = {
    b: 1,
    kb: 1e3,
    mb: 1e6,
    gb: 1e9,
    tb: 1e12,
    kib: 1024,
    mib: 1024 ** 2,
    gib: 1024 ** 3,
    tib: 1024 ** 4,
  }[unit] ?? 1;
  return Math.round(n * scale);
}

function parseMemUsage(value) {
  const [used, limit] = String(value ?? '').split('/').map((part) => part.trim());
  return {
    usedBytes: parseBytes(used),
    limitBytes: parseBytes(limit),
  };
}

function resolveRuntimeMode() {
  const configured = String(process.env.GOOSED_RUNTIME ?? 'auto').trim().toLowerCase();
  if (configured === 'native' || configured === 'docker') return configured;
  try {
    const line = sh("docker ps --filter 'label=com.docker.compose.project=goosed-prod' --format '{{.Names}}' 2>/dev/null | head -1");
    return line ? 'docker' : 'native';
  } catch {
    return 'native';
  }
}

function portFromTarget(target) {
  try {
    return new URL(target).port || '443';
  } catch {
    return null;
  }
}

function targetWorkerIds() {
  const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
    .split(',')
    .map((value) => value.trim())
    .filter(Boolean);
  return targets.map((target, index) => {
    const port = portFromTarget(target);
    return {
      id: `goosed-${index + 1}`,
      target,
      port,
      container: `goosed-prod-${index + 1}`,
      launchdLabel: port ? `com.tkmind.local-goosed-${port}` : null,
    };
  });
}

function readDockerStats(container) {
  try {
    const line = sh(`docker stats --no-stream --format '{{json .}}' ${JSON.stringify(container)} 2>/dev/null`);
    return line ? JSON.parse(line) : null;
  } catch {
    return null;
  }
}

function readDockerInspect(container) {
  try {
    const line = sh(`docker inspect --format '{{json .State}}' ${JSON.stringify(container)} 2>/dev/null`);
    return line ? JSON.parse(line) : null;
  } catch {
    return null;
  }
}

function readContainerFdCount(container) {
  try {
    return Number(sh(`docker exec ${JSON.stringify(container)} sh -lc 'ls /proc/1/fd 2>/dev/null | wc -l'`)) || 0;
  } catch {
    return 0;
  }
}

function readListenPid(port) {
  if (!port) return null;
  try {
    const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
    const n = Number(pid);
    return Number.isFinite(n) && n > 0 ? n : null;
  } catch {
    return null;
  }
}

function readNativeFdCount(pid) {
  if (!pid) return 0;
  try {
    return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
  } catch {
    return 0;
  }
}

function readNativeProcessStats(pid) {
  if (!pid) return null;
  try {
    const line = sh(`ps -p ${pid} -o %cpu=,rss= 2>/dev/null | tail -1`);
    const match = line.trim().match(/^([\d.]+)\s+(\d+)$/);
    if (!match) return null;
    return {
      cpuLoad: Number(match[1]) || 0,
      rssBytes: (Number(match[2]) || 0) * 1024,
    };
  } catch {
    return null;
  }
}

function readLaunchdState(label) {
  if (!label) return null;
  try {
    return sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`) || null;
  } catch {
    return null;
  }
}

function sampleDockerWorker(worker, fdWarn, now) {
  const stats = readDockerStats(worker.container);
  const state = readDockerInspect(worker.container);
  const fdCount = readContainerFdCount(worker.container);
  const mem = parseMemUsage(stats?.MemUsage);
  const cpuLoad = parsePercent(stats?.CPUPerc);
  const memoryPressure = parsePercent(stats?.MemPerc);
  const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
  return {
    ...worker,
    runtime: 'docker',
    ok: Boolean(stats && state?.Running),
    health: state?.Health?.Status ?? state?.Status ?? null,
    hostPid: Number(state?.Pid ?? 0) || null,
    cpuLoad,
    memoryPressure,
    fdPressure,
    fdCount,
    pids: Number(stats?.PIDs ?? 0),
    memUsedBytes: mem.usedBytes,
    memLimitBytes: mem.limitBytes,
    sampledAt: now,
  };
}

function sampleNativeWorker(worker, fdWarn, now) {
  const hostPid = readListenPid(worker.port);
  const fdCount = readNativeFdCount(hostPid);
  const proc = readNativeProcessStats(hostPid);
  const launchdState = readLaunchdState(worker.launchdLabel);
  const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
  const memUsedBytes = proc?.rssBytes ?? 0;
  return {
    ...worker,
    runtime: 'native',
    ok: Boolean(hostPid && launchdState === 'running'),
    health: launchdState,
    hostPid,
    cpuLoad: proc?.cpuLoad ?? 0,
    memoryPressure: 0,
    fdPressure,
    fdCount,
    pids: hostPid ? 1 : 0,
    memUsedBytes,
    memLimitBytes: 0,
    sampledAt: now,
    container: worker.launchdLabel ?? worker.container,
  };
}

function workerKey(namespace, id, field) {
  return [namespace, 'worker', id, field].join(':');
}

loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));

const runtimeMode = resolveRuntimeMode();
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
const action = process.argv[2] || 'sample';
const dryRun = process.argv.includes('--dry-run') || action === 'status';
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
const now = Date.now();

if (!['sample', 'status'].includes(action)) {
  console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
  console.error('Env: GOOSED_RUNTIME=auto|native|docker (default auto)');
  process.exit(2);
}

const workers = targetWorkerIds().map((worker) => (
  runtimeMode === 'native'
    ? sampleNativeWorker(worker, fdWarn, now)
    : sampleDockerWorker(worker, fdWarn, now)
));

if (!dryRun) {
  const client = createClient({ url: redisUrl });
  client.on('error', (err) => {
    console.error(`Redis error: ${err instanceof Error ? err.message : err}`);
  });
  await client.connect();
  for (const worker of workers) {
    await client
      .multi()
      .set(workerKey(namespace, worker.id, 'cpu_load'), String(worker.cpuLoad))
      .set(workerKey(namespace, worker.id, 'memory_pressure'), String(worker.memoryPressure))
      .set(workerKey(namespace, worker.id, 'fd_pressure'), String(worker.fdPressure))
      .set(workerKey(namespace, worker.id, 'fd_count'), String(worker.fdCount))
      .set(workerKey(namespace, worker.id, 'container_pids'), String(worker.pids))
      .set(workerKey(namespace, worker.id, 'container_name'), worker.container)
      .set(workerKey(namespace, worker.id, 'container_health'), worker.health ?? '')
      .set(workerKey(namespace, worker.id, 'container_host_pid'), String(worker.hostPid ?? ''))
      .set(workerKey(namespace, worker.id, 'metrics_sampled_at'), String(worker.sampledAt))
      .set(workerKey(namespace, worker.id, 'heartbeat'), String(now), { EX: 90 })
      .exec();
  }
  await client.quit();
}

console.log(JSON.stringify({
  ok: workers.every((worker) => worker.ok),
  action,
  dryRun,
  runtimeMode,
  namespace,
  redisWrites: !dryRun,
  workers,
}, null, 2));

process.exit(workers.every((worker) => worker.ok) ? 0 : 1);