memind/scripts/runtime-slo-report.mjs

#!/usr/bin/env node
import fs from 'node:fs';
import path from 'node:path';
import { Agent, fetch as undiciFetch } from 'undici';
import mysql from 'mysql2/promise';
import { createClient } from 'redis';

function loadEnvFile(filePath) {
  if (!fs.existsSync(filePath)) return;
  for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
    const trimmed = line.trim();
    if (!trimmed || trimmed.startsWith('#')) continue;
    const idx = trimmed.indexOf('=');
    if (idx < 0) continue;
    const key = trimmed.slice(0, idx).trim();
    const value = trimmed.slice(idx + 1).trim();
    if (!process.env[key]) process.env[key] = value;
  }
}

function parseMysqlConfig() {
  if (process.env.DATABASE_URL) {
    const url = new URL(process.env.DATABASE_URL);
    if (url.protocol !== 'mysql:') {
      throw new Error(`Unsupported DATABASE_URL scheme for SLO report: ${url.protocol}`);
    }
    return {
      host: url.hostname,
      port: Number(url.port || 3306),
      user: decodeURIComponent(url.username),
      password: decodeURIComponent(url.password),
      database: url.pathname.replace(/^\/+/, ''),
      charset: 'utf8mb4',
    };
  }
  return {
    host: process.env.MYSQL_HOST,
    port: Number(process.env.MYSQL_PORT || 3306),
    user: process.env.MYSQL_USER,
    password: process.env.MYSQL_PASSWORD,
    database: process.env.MYSQL_DATABASE,
    charset: 'utf8mb4',
  };
}

function walkCount(root) {
  const stack = [root];
  let files = 0;
  let dirs = 0;
  let bytes = 0;
  while (stack.length) {
    const current = stack.pop();
    let entries = [];
    try {
      entries = fs.readdirSync(current, { withFileTypes: true });
    } catch {
      continue;
    }
    for (const entry of entries) {
      const full = path.join(current, entry.name);
      if (entry.isDirectory()) {
        dirs += 1;
        stack.push(full);
        continue;
      }
      if (!entry.isFile()) continue;
      files += 1;
      try {
        bytes += fs.statSync(full).size;
      } catch {
        // ignore files that disappear while counting
      }
    }
  }
  return { files, dirs, bytes };
}

function uniquePaths(paths) {
  const seen = new Set();
  const result = [];
  for (const item of paths) {
    const resolved = path.resolve(item);
    if (seen.has(resolved)) continue;
    seen.add(resolved);
    result.push(resolved);
  }
  return result;
}

function parseArgs(argv) {
  const args = {
    writeReport: false,
    prune: false,
    retentionDays: Number(process.env.MEMIND_RUNTIME_SLO_RETENTION_DAYS || 30),
    reportDir: process.env.MEMIND_RUNTIME_REPORT_DIR || '',
  };
  for (let i = 0; i < argv.length; i += 1) {
    const item = argv[i];
    if (item === '--write-report') args.writeReport = true;
    else if (item === '--prune') args.prune = true;
    else if (item === '--retention-days') args.retentionDays = Number(argv[++i] ?? args.retentionDays);
    else if (item === '--report-dir') args.reportDir = String(argv[++i] ?? args.reportDir);
    else if (item === '--help' || item === '-h') args.help = true;
  }
  if (!Number.isFinite(args.retentionDays) || args.retentionDays < 1) {
    throw new Error(`Invalid --retention-days: ${args.retentionDays}`);
  }
  return args;
}

function printHelp() {
  console.log([
    'Usage:',
    '  node scripts/runtime-slo-report.mjs [--write-report] [--prune] [--retention-days <days>] [--report-dir <dir>]',
    '',
    'Default behavior is read-only and writes nothing.',
    '--write-report writes JSON and Markdown snapshots to an operations report directory.',
    '--prune deletes old .json/.md snapshots in the report directory only.',
  ].join('\n'));
}

function pruneReportDir(reportDir, retentionDays) {
  const resolvedDir = path.resolve(reportDir);
  const cutoffMs = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
  const result = {
    enabled: true,
    reportDir: resolvedDir,
    retentionDays,
    deleted: [],
    kept: 0,
    skipped: 0,
    errors: [],
  };
  let entries = [];
  try {
    entries = fs.readdirSync(resolvedDir, { withFileTypes: true });
  } catch (err) {
    result.errors.push(err instanceof Error ? err.message : String(err));
    return result;
  }
  for (const entry of entries) {
    if (!entry.isFile()) {
      result.skipped += 1;
      continue;
    }
    if (!entry.name.endsWith('.json') && !entry.name.endsWith('.md')) {
      result.skipped += 1;
      continue;
    }
    const full = path.join(resolvedDir, entry.name);
    let stat = null;
    try {
      stat = fs.statSync(full);
    } catch (err) {
      result.errors.push(`${entry.name}: ${err instanceof Error ? err.message : String(err)}`);
      continue;
    }
    if (stat.mtimeMs >= cutoffMs) {
      result.kept += 1;
      continue;
    }
    try {
      fs.unlinkSync(full);
      result.deleted.push(entry.name);
    } catch (err) {
      result.errors.push(`${entry.name}: ${err instanceof Error ? err.message : String(err)}`);
    }
  }
  return result;
}

function tableCountQueries() {
  return [
    ['h5_users', 'users'],
    ['h5_user_sessions', 'sessions'],
    ['h5_agent_runs', 'agentRuns'],
    ['h5_agent_run_events', 'agentRunEvents'],
    ['h5_capability_grants', 'capabilityGrants'],
    ['h5_llm_provider_keys', 'llmProviderKeys'],
    ['mindspace_assets', 'mindspaceAssets'],
    ['mindspace_pages', 'mindspacePages'],
    ['mindspace_publications', 'mindspacePublications'],
  ];
}

async function safeTableCount(conn, table) {
  try {
    const [rows] = await conn.query(`SELECT COUNT(*) AS count FROM \`${table}\``);
    return Number(rows[0]?.count ?? 0);
  } catch (err) {
    return { error: err instanceof Error ? err.message : String(err) };
  }
}

async function readDbSummary() {
  const conn = await mysql.createConnection(parseMysqlConfig());
  try {
    const result = {};
    for (const [table, key] of tableCountQueries()) {
      result[key] = await safeTableCount(conn, table);
    }
    return result;
  } finally {
    await conn.end();
  }
}

async function readRedisSummary(namespace, redisUrl) {
  const client = createClient({ url: redisUrl });
  await client.connect();
  try {
    const workerKeys = await client.keys(`${namespace}:worker:*:active_streams`);
    const streamKeys = await client.keys(`${namespace}:stream:*:status`);
    const sessionKeys = await client.keys(`${namespace}:session:*:target`);
    return {
      workerCount: workerKeys.length,
      streamStatusCount: streamKeys.length,
      sessionTargetCount: sessionKeys.length,
    };
  } finally {
    await client.quit();
  }
}

function summarizeRuntime(runtimeJson) {
  const workers = runtimeJson?.router?.workers ?? [];
  const toolQueue = runtimeJson?.toolRuntime?.queue ?? null;
  const staleMetricMs = 2 * 60 * 1000;
  const staleHeartbeatMs = Number(process.env.MEMIND_RUNTIME_HEARTBEAT_STALE_MS || 60 * 1000);
  const now = Date.now();
  return {
    routerEnabled: Boolean(runtimeJson?.router?.enabled),
    publicBaseUrl: runtimeJson?.publicBaseUrl ?? null,
    toolRuntime: runtimeJson?.toolRuntime ?? null,
    toolQueue,
    toolQueueSlo: toolQueue
      ? {
          heartbeatMs: Number(toolQueue.heartbeatMs ?? 0),
          maxConcurrentRuns: Number(toolQueue.maxConcurrentRuns ?? 0),
          statusCounts: toolQueue.statusCounts ?? {},
          oldestRunningStartedAt: toolQueue.oldestRunningStartedAt ?? null,
          oldestRunningAgeMs: Number(toolQueue.oldestRunningAgeMs ?? 0),
          oldestRunningHeartbeatAt: toolQueue.oldestRunningHeartbeatAt ?? null,
          oldestRunningHeartbeatAgeMs: Number(toolQueue.oldestRunningHeartbeatAgeMs ?? 0),
          runningWithoutHeartbeatCount: Number(toolQueue.runningWithoutHeartbeatCount ?? 0),
          latestRunningRun: toolQueue.latestRunningRun ?? null,
        }
      : null,
    workers: workers.map((worker) => ({
      id: worker.id,
      healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null,
      activeStreams: worker.activeStreams,
      streamOpenCount: worker.streamOpenCount,
      streamAbortCount: worker.streamAbortCount,
      streamErrorCount: worker.streamErrorCount,
      ewmaFirstTokenMs: worker.ewmaFirstTokenMs,
      lastFirstTokenMs: worker.lastFirstTokenMs,
      lastFirstTokenAt: worker.lastFirstTokenAt,
      firstTokenCount: worker.firstTokenCount,
      firstToken5m: worker.firstToken5m ?? null,
      firstToken1h: worker.firstToken1h ?? null,
      cpuLoad: worker.cpuLoad,
      memoryPressure: worker.memoryPressure,
      fdPressure: worker.fdPressure,
      fdCount: worker.fdCount,
      containerHealth: worker.containerHealth,
      heartbeat: worker.heartbeat,
      heartbeatAgeMs: worker.heartbeat ? now - worker.heartbeat : null,
      heartbeatFresh: worker.heartbeat ? now - worker.heartbeat <= staleHeartbeatMs : false,
      heartbeatSource: worker.heartbeatSource ?? null,
      heartbeatOk: worker.heartbeatOk ?? null,
      heartbeatStatusCode: worker.heartbeatStatusCode ?? null,
      heartbeatLatencyMs: worker.heartbeatLatencyMs ?? null,
      heartbeatError: worker.heartbeatError ?? null,
      metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null,
      metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false,
      score: worker.score,
      drain: worker.drain,
    })),
  };
}

const envFile = process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env');
const appRoot = process.env.MEMIND_APP_ROOT || path.dirname(path.resolve(envFile));
loadEnvFile(envFile);
const args = parseArgs(process.argv.slice(2));
if (args.help) {
  printHelp();
  process.exit(0);
}

const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://m.tkmind.cn').replace(/\/$/, '');
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
const mindSpaceRoots = uniquePaths([
  path.join(appRoot, 'MindSpace'),
  process.env.MINDSPACE_STORAGE_ROOT || path.join(appRoot, 'data', 'mindspace'),
]);
const dispatcher = new Agent({ connect: { rejectUnauthorized: false } });

async function fetchRuntime() {
  const res = await undiciFetch(`${publicBase}/api/runtime/status`, {
    dispatcher: publicBase.startsWith('https://127.0.0.1') ? dispatcher : undefined,
  });
  const text = await res.text();
  return {
    ok: res.ok,
    status: res.status,
    json: JSON.parse(text),
  };
}

const runtime = await fetchRuntime().catch((err) => ({ ok: false, error: err.message }));
const runtimeSummary = runtime.ok ? summarizeRuntime(runtime.json) : null;
const db = await readDbSummary().catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
const redis = await readRedisSummary(namespace, redisUrl).catch((err) => ({
  error: err instanceof Error ? err.message : String(err),
}));
const mindSpace = mindSpaceRoots.map((root) => (
  fs.existsSync(root)
    ? { root, ...walkCount(root) }
    : { root, error: 'missing' }
));

const failures = [];
if (!runtime.ok) failures.push('runtime_status_unavailable');
if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled');
for (const worker of runtimeSummary?.workers ?? []) {
  if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`);
  if (!worker.heartbeatFresh) failures.push(`${worker.id}_heartbeat_stale`);
  if (worker.heartbeatOk === false) failures.push(`${worker.id}_heartbeat_unhealthy`);
  if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`);
  if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`);
  if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`);
  if (worker.firstTokenCount > 0 && worker.ewmaFirstTokenMs <= 0) failures.push(`${worker.id}_first_token_ewma_missing`);
}
if (runtimeSummary?.toolRuntime?.chatInjectsCodeTools !== false) failures.push('chat_injects_code_tools');
if (runtimeSummary?.toolQueue?.error) failures.push('tool_queue_status_error');
if (runtimeSummary?.toolQueue?.inFlight > runtimeSummary?.toolQueue?.maxConcurrentRuns) {
  failures.push('tool_queue_concurrency_exceeded');
}
const agentRunHeartbeatStaleMs = Number(
  process.env.MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS ||
    Math.max(90 * 1000, Number(runtimeSummary?.toolQueueSlo?.heartbeatMs ?? 0) * 3),
);
if (
  runtimeSummary?.toolQueueSlo?.oldestRunningHeartbeatAgeMs > agentRunHeartbeatStaleMs
) {
  failures.push(`agent_run_heartbeat_stale_${runtimeSummary.toolQueueSlo.oldestRunningHeartbeatAgeMs}`);
}
if (
  Number(runtimeSummary?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0) > 0 &&
  Number(runtimeSummary?.toolQueueSlo?.oldestRunningAgeMs ?? 0) > agentRunHeartbeatStaleMs
) {
  failures.push(`agent_run_missing_heartbeat_${runtimeSummary.toolQueueSlo.runningWithoutHeartbeatCount}`);
}

const reportDir = path.resolve(args.reportDir || path.join(appRoot, 'reports', 'runtime-slo'));
if (args.writeReport || args.prune) {
  fs.mkdirSync(reportDir, { recursive: true });
}
const reportPrune = args.prune
  ? pruneReportDir(reportDir, args.retentionDays)
  : { enabled: false, reportDir, retentionDays: args.retentionDays };
for (const error of reportPrune.errors ?? []) {
  failures.push(`report_prune_error:${error}`);
}

const report = {
  ok: failures.length === 0,
  checkedAt: new Date().toISOString(),
  publicBase,
  appRoot,
  envFile,
  namespace,
  runtime: runtimeSummary,
  db,
  redis,
  mindSpace,
  failures,
  writes: {
    database: false,
    mindSpace: false,
    redis: false,
    report: Boolean(args.writeReport),
    reportPrune: Boolean(args.prune),
  },
  reportPrune,
};

function markdownReport(payload) {
  const workers = payload.runtime?.workers ?? [];
  const lines = [
    `# Memind Runtime SLO Report`,
    '',
    `- checkedAt: ${payload.checkedAt}`,
    `- ok: ${payload.ok}`,
    `- failures: ${payload.failures.length ? payload.failures.join(', ') : 'none'}`,
    `- publicBase: ${payload.publicBase}`,
    `- appRoot: ${payload.appRoot}`,
    '',
    '## Workers',
    '',
    '| worker | healthy | active | errors | first-token 5m p50/p95 | first-token 1h p50/p95 | heartbeat | metricsFresh | score |',
    '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',
  ];
  for (const worker of workers) {
    lines.push([
      worker.id,
      worker.healthy,
      worker.activeStreams,
      worker.streamErrorCount,
      `${worker.firstToken5m?.p50Ms ?? 0}/${worker.firstToken5m?.p95Ms ?? 0}`,
      `${worker.firstToken1h?.p50Ms ?? 0}/${worker.firstToken1h?.p95Ms ?? 0}`,
      `${worker.heartbeatSource ?? 'none'}:${worker.heartbeatFresh}`,
      worker.metricsFresh,
      worker.score,
    ].join(' | ').replace(/^/, '| ').replace(/$/, ' |'));
  }
  lines.push(
    '',
    '## Tool Queue',
    '',
    `- heartbeatMs: ${payload.runtime?.toolQueueSlo?.heartbeatMs ?? 0}`,
    `- oldestRunningHeartbeatAgeMs: ${payload.runtime?.toolQueueSlo?.oldestRunningHeartbeatAgeMs ?? 0}`,
    `- runningWithoutHeartbeatCount: ${payload.runtime?.toolQueueSlo?.runningWithoutHeartbeatCount ?? 0}`,
    '',
    '```json',
    JSON.stringify(payload.runtime?.toolQueue ?? null, null, 2),
    '```',
    '',
    '## Writes',
    '',
    '```json',
    JSON.stringify(payload.writes, null, 2),
    '```',
    '',
    '## Report Prune',
    '',
    '```json',
    JSON.stringify(payload.reportPrune ?? null, null, 2),
    '```',
    '',
  );
  return lines.join('\n');
}

if (args.writeReport) {
  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
  report.reportFiles = {
    json: path.join(reportDir, `${stamp}.json`),
    markdown: path.join(reportDir, `${stamp}.md`),
  };
  fs.writeFileSync(report.reportFiles.json, `${JSON.stringify(report, null, 2)}\n`);
  fs.writeFileSync(report.reportFiles.markdown, markdownReport(report));
}

console.log(JSON.stringify(report, null, 2));
process.exit(report.ok ? 0 : 1);