#!/usr/bin/env node import fs from 'node:fs'; import path from 'node:path'; import { Agent, fetch as undiciFetch } from 'undici'; import mysql from 'mysql2/promise'; import { createClient } from 'redis'; function loadEnvFile(filePath) { if (!fs.existsSync(filePath)) return; for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#')) continue; const idx = trimmed.indexOf('='); if (idx < 0) continue; const key = trimmed.slice(0, idx).trim(); const value = trimmed.slice(idx + 1).trim(); if (!process.env[key]) process.env[key] = value; } } function parseMysqlConfig() { if (process.env.DATABASE_URL) { const url = new URL(process.env.DATABASE_URL); if (url.protocol !== 'mysql:') { throw new Error(`Unsupported DATABASE_URL scheme for SLO report: ${url.protocol}`); } return { host: url.hostname, port: Number(url.port || 3306), user: decodeURIComponent(url.username), password: decodeURIComponent(url.password), database: url.pathname.replace(/^\/+/, ''), charset: 'utf8mb4', }; } return { host: process.env.MYSQL_HOST, port: Number(process.env.MYSQL_PORT || 3306), user: process.env.MYSQL_USER, password: process.env.MYSQL_PASSWORD, database: process.env.MYSQL_DATABASE, charset: 'utf8mb4', }; } function walkCount(root) { const stack = [root]; let files = 0; let dirs = 0; let bytes = 0; while (stack.length) { const current = stack.pop(); let entries = []; try { entries = fs.readdirSync(current, { withFileTypes: true }); } catch { continue; } for (const entry of entries) { const full = path.join(current, entry.name); if (entry.isDirectory()) { dirs += 1; stack.push(full); continue; } if (!entry.isFile()) continue; files += 1; try { bytes += fs.statSync(full).size; } catch { // ignore files that disappear while counting } } } return { files, dirs, bytes }; } function uniquePaths(paths) { const seen = new Set(); const result = []; for (const item of paths) { const resolved = path.resolve(item); if (seen.has(resolved)) continue; seen.add(resolved); result.push(resolved); } return result; } function parseArgs(argv) { const args = { writeReport: false, reportDir: process.env.MEMIND_RUNTIME_REPORT_DIR || '', }; for (let i = 0; i < argv.length; i += 1) { const item = argv[i]; if (item === '--write-report') args.writeReport = true; else if (item === '--report-dir') args.reportDir = String(argv[++i] ?? args.reportDir); else if (item === '--help' || item === '-h') args.help = true; } return args; } function printHelp() { console.log([ 'Usage:', ' node scripts/runtime-slo-report.mjs [--write-report] [--report-dir ]', '', 'Default behavior is read-only and writes nothing.', '--write-report writes JSON and Markdown snapshots to an operations report directory.', ].join('\n')); } function tableCountQueries() { return [ ['h5_users', 'users'], ['h5_user_sessions', 'sessions'], ['h5_agent_runs', 'agentRuns'], ['h5_agent_run_events', 'agentRunEvents'], ['h5_capability_grants', 'capabilityGrants'], ['h5_llm_provider_keys', 'llmProviderKeys'], ['mindspace_assets', 'mindspaceAssets'], ['mindspace_pages', 'mindspacePages'], ['mindspace_publications', 'mindspacePublications'], ]; } async function safeTableCount(conn, table) { try { const [rows] = await conn.query(`SELECT COUNT(*) AS count FROM \`${table}\``); return Number(rows[0]?.count ?? 0); } catch (err) { return { error: err instanceof Error ? err.message : String(err) }; } } async function readDbSummary() { const conn = await mysql.createConnection(parseMysqlConfig()); try { const result = {}; for (const [table, key] of tableCountQueries()) { result[key] = await safeTableCount(conn, table); } return result; } finally { await conn.end(); } } async function readRedisSummary(namespace, redisUrl) { const client = createClient({ url: redisUrl }); await client.connect(); try { const workerKeys = await client.keys(`${namespace}:worker:*:active_streams`); const streamKeys = await client.keys(`${namespace}:stream:*:status`); const sessionKeys = await client.keys(`${namespace}:session:*:target`); return { workerCount: workerKeys.length, streamStatusCount: streamKeys.length, sessionTargetCount: sessionKeys.length, }; } finally { await client.quit(); } } function summarizeRuntime(runtimeJson) { const workers = runtimeJson?.router?.workers ?? []; const staleMetricMs = 2 * 60 * 1000; const now = Date.now(); return { routerEnabled: Boolean(runtimeJson?.router?.enabled), publicBaseUrl: runtimeJson?.publicBaseUrl ?? null, toolRuntime: runtimeJson?.toolRuntime ?? null, toolQueue: runtimeJson?.toolRuntime?.queue ?? null, workers: workers.map((worker) => ({ id: worker.id, healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null, activeStreams: worker.activeStreams, streamOpenCount: worker.streamOpenCount, streamAbortCount: worker.streamAbortCount, streamErrorCount: worker.streamErrorCount, ewmaFirstTokenMs: worker.ewmaFirstTokenMs, lastFirstTokenMs: worker.lastFirstTokenMs, lastFirstTokenAt: worker.lastFirstTokenAt, firstTokenCount: worker.firstTokenCount, firstToken5m: worker.firstToken5m ?? null, firstToken1h: worker.firstToken1h ?? null, cpuLoad: worker.cpuLoad, memoryPressure: worker.memoryPressure, fdPressure: worker.fdPressure, fdCount: worker.fdCount, containerHealth: worker.containerHealth, metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null, metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false, score: worker.score, drain: worker.drain, })), }; } const envFile = process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'); const appRoot = process.env.MEMIND_APP_ROOT || path.dirname(path.resolve(envFile)); loadEnvFile(envFile); const args = parseArgs(process.argv.slice(2)); if (args.help) { printHelp(); process.exit(0); } const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, ''); const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0'; const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime'; const mindSpaceRoots = uniquePaths([ path.join(appRoot, 'MindSpace'), process.env.MINDSPACE_STORAGE_ROOT || path.join(appRoot, 'data', 'mindspace'), ]); const dispatcher = new Agent({ connect: { rejectUnauthorized: false } }); async function fetchRuntime() { const res = await undiciFetch(`${publicBase}/api/runtime/status`, { dispatcher: publicBase.startsWith('https://127.0.0.1') ? dispatcher : undefined, }); const text = await res.text(); return { ok: res.ok, status: res.status, json: JSON.parse(text), }; } const runtime = await fetchRuntime().catch((err) => ({ ok: false, error: err.message })); const runtimeSummary = runtime.ok ? summarizeRuntime(runtime.json) : null; const db = await readDbSummary().catch((err) => ({ error: err instanceof Error ? err.message : String(err) })); const redis = await readRedisSummary(namespace, redisUrl).catch((err) => ({ error: err instanceof Error ? err.message : String(err), })); const mindSpace = mindSpaceRoots.map((root) => ( fs.existsSync(root) ? { root, ...walkCount(root) } : { root, error: 'missing' } )); const failures = []; if (!runtime.ok) failures.push('runtime_status_unavailable'); if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled'); for (const worker of runtimeSummary?.workers ?? []) { if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`); if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`); if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`); if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`); if (worker.firstTokenCount > 0 && worker.ewmaFirstTokenMs <= 0) failures.push(`${worker.id}_first_token_ewma_missing`); } if (runtimeSummary?.toolRuntime?.chatInjectsCodeTools !== false) failures.push('chat_injects_code_tools'); if (runtimeSummary?.toolQueue?.error) failures.push('tool_queue_status_error'); if (runtimeSummary?.toolQueue?.inFlight > runtimeSummary?.toolQueue?.maxConcurrentRuns) { failures.push('tool_queue_concurrency_exceeded'); } const report = { ok: failures.length === 0, checkedAt: new Date().toISOString(), publicBase, appRoot, envFile, namespace, runtime: runtimeSummary, db, redis, mindSpace, failures, writes: { database: false, mindSpace: false, redis: false, report: Boolean(args.writeReport), }, }; function markdownReport(payload) { const workers = payload.runtime?.workers ?? []; const lines = [ `# Memind Runtime SLO Report`, '', `- checkedAt: ${payload.checkedAt}`, `- ok: ${payload.ok}`, `- failures: ${payload.failures.length ? payload.failures.join(', ') : 'none'}`, `- publicBase: ${payload.publicBase}`, `- appRoot: ${payload.appRoot}`, '', '## Workers', '', '| worker | healthy | active | errors | first-token 5m p50/p95 | first-token 1h p50/p95 | metricsFresh | score |', '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ]; for (const worker of workers) { lines.push([ worker.id, worker.healthy, worker.activeStreams, worker.streamErrorCount, `${worker.firstToken5m?.p50Ms ?? 0}/${worker.firstToken5m?.p95Ms ?? 0}`, `${worker.firstToken1h?.p50Ms ?? 0}/${worker.firstToken1h?.p95Ms ?? 0}`, worker.metricsFresh, worker.score, ].join(' | ').replace(/^/, '| ').replace(/$/, ' |')); } lines.push( '', '## Tool Queue', '', '```json', JSON.stringify(payload.runtime?.toolQueue ?? null, null, 2), '```', '', '## Writes', '', '```json', JSON.stringify(payload.writes, null, 2), '```', '', ); return lines.join('\n'); } if (args.writeReport) { const reportDir = path.resolve(args.reportDir || path.join(appRoot, 'reports', 'runtime-slo')); fs.mkdirSync(reportDir, { recursive: true }); const stamp = new Date().toISOString().replace(/[:.]/g, '-'); report.reportFiles = { json: path.join(reportDir, `${stamp}.json`), markdown: path.join(reportDir, `${stamp}.md`), }; fs.writeFileSync(report.reportFiles.json, `${JSON.stringify(report, null, 2)}\n`); fs.writeFileSync(report.reportFiles.markdown, markdownReport(report)); } console.log(JSON.stringify(report, null, 2)); process.exit(report.ok ? 0 : 1);