memind/scripts/verify-h5-llm-router-shadow.mjs

#!/usr/bin/env node
/**
 * Local smoke: verify ambiguous H5 chat triggers LLM router shadow logging.
 * Usage: node scripts/verify-h5-llm-router-shadow.mjs
 */
import crypto from 'node:crypto';
import { loadH5Environment } from './load-env.mjs';
import { createDbPool } from '../db.mjs';
import { createUserAuth } from '../user-auth.mjs';
import { resolveChatIntentRouterPolicy } from '../chat-intent-router.mjs';

loadH5Environment(import.meta.dirname);

const PORTAL = `http://127.0.0.1:${process.env.H5_PORT ?? 8081}`;
const USERNAME = process.env.VERIFY_LLM_ROUTER_USER ?? 'john2';
const PASSWORD = process.env.JOHN_PASSWORD ?? process.env.H5_ACCESS_PASSWORD ?? '888888';
const QUERY = '周末想放松一下，有什么活动建议？';
const MAX_WAIT_MS = 120_000;
const LOG_PATH = process.env.MEMIND_LLM_ROUTER_SHADOW_LOG
  ?? '/Users/john/Library/Logs/memind-h5-local.log';

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function pass(label, detail = '') {
  console.log(`PASS  ${label}${detail ? `: ${detail}` : ''}`);
}

function fail(label, detail = '') {
  console.error(`FAIL  ${label}${detail ? `: ${detail}` : ''}`);
  process.exitCode = 1;
}

async function login() {
  const pool = await createDbPool();
  const auth = createUserAuth(pool);
  const result = await auth.login({ username: USERNAME, password: PASSWORD, ip: '127.0.0.1' });
  if (!result.ok) {
    throw new Error(`john 登录失败: ${result.message ?? 'unknown'}`);
  }
  await pool.end();
  return result.token;
}

async function createRun(token) {
  const requestId = crypto.randomUUID();
  const response = await fetch(`${PORTAL}/api/agent/runs`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Cookie: `tkmind_user_session=${token}`,
    },
    body: JSON.stringify({
      request_id: requestId,
      user_message: {
        id: crypto.randomUUID(),
        role: 'user',
        content: [{ type: 'text', text: QUERY }],
        metadata: { userVisible: true, displayText: QUERY },
      },
    }),
  });
  const body = await response.json().catch(() => ({}));
  if (!response.ok) {
    throw new Error(`POST /agent/runs ${response.status}: ${JSON.stringify(body)}`);
  }
  const run = body.run ?? body;
  return { runId: run.id, requestId, status: run.status };
}

async function waitForRun(token, runId) {
  const started = Date.now();
  while (Date.now() - started < MAX_WAIT_MS) {
    const response = await fetch(`${PORTAL}/api/agent/runs/${runId}`, {
      headers: { Cookie: `tkmind_user_session=${token}` },
    });
    const payload = await response.json().catch(() => ({}));
    const body = payload.run ?? payload;
    if (!response.ok) {
      throw new Error(`GET /agent/runs/${runId} ${response.status}: ${JSON.stringify(payload)}`);
    }
    if (['succeeded', 'failed'].includes(body.status)) {
      return body;
    }
    await sleep(1000);
  }
  throw new Error(`run ${runId} 未在 ${MAX_WAIT_MS}ms 内完成`);
}

async function loadRunEvents(runId) {
  const pool = await createDbPool();
  const [rows] = await pool.query(
    `SELECT event_type, data_json FROM h5_agent_run_events WHERE run_id = ? ORDER BY id ASC`,
    [runId],
  );
  await pool.end();
  return rows.map((row) => ({
    event_type: row.event_type,
    data: typeof row.data_json === 'string' ? JSON.parse(row.data_json) : row.data_json,
  }));
}

async function waitForPortal() {
  const started = Date.now();
  while (Date.now() - started < 60_000) {
    try {
      const response = await fetch(`${PORTAL}/api/status`);
      if (response.ok) return;
    } catch {
      // retry
    }
    await sleep(1000);
  }
  throw new Error(`Portal ${PORTAL} 未在 60s 内就绪`);
}

async function readShadowLogLines() {
  try {
    const fs = await import('node:fs/promises');
    const text = await fs.readFile(LOG_PATH, 'utf8');
    return text.split('\n').filter((line) => line.includes('[chat-llm-router-shadow]'));
  } catch {
    return [];
  }
}

async function main() {
  const policy = resolveChatIntentRouterPolicy();
  console.log('--- router policy ---');
  console.log(JSON.stringify({
    enabled: policy.enabled,
    shadowMode: policy.shadowMode,
    canaryUserIds: policy.canaryUserIds,
    timeoutMs: policy.timeoutMs,
  }, null, 2));

  if (!policy.enabled || !policy.shadowMode) {
    fail('policy', '需要 MEMIND_CHAT_LLM_ROUTER_ENABLED=1 且 MEMIND_CHAT_LLM_ROUTER_SHADOW=1');
    return;
  }
  pass('policy', 'shadow 模式已开启');

  await waitForPortal();
  pass('portal', PORTAL);

  const token = await login();
  pass('login', USERNAME);

  const beforeShadowLines = await readShadowLogLines();
  const { runId } = await createRun(token);
  pass('agent run', runId);

  await waitForRun(token, runId);
  pass('run finished');

  const events = await loadRunEvents(runId);
  const routed = events.find((row) => row.event_type === 'intent_routed');
  if (!routed?.data) {
    fail('intent_routed', '未找到路由事件');
  } else {
    const { route, source, reason, llmShadow } = routed.data;
    console.log('\n--- intent_routed ---');
    console.log(JSON.stringify(routed.data, null, 2));
    if (source === 'fallback' && route === 'agent_orchestration') {
      pass('shadow routing', '规则未命中，Shadow 仍 fallback Agent');
    } else if (source === 'rule') {
      fail('shadow routing', `意外命中规则: ${reason ?? route}`);
    } else {
      fail('shadow routing', `source=${source}, route=${route}`);
    }
    if (llmShadow) {
      pass('llm shadow payload', JSON.stringify(llmShadow));
    } else {
      fail('llm shadow payload', 'intent_routed 缺少 llmShadow（LLM 路由可能未启用）');
    }
  }

  await sleep(500);
  const afterShadowLines = await readShadowLogLines();
  const newShadowLines = afterShadowLines.slice(beforeShadowLines.length);
  if (newShadowLines.length > 0) {
    pass('shadow log', `${newShadowLines.length} 条 [chat-llm-router-shadow]`);
    for (const line of newShadowLines.slice(-3)) {
      console.log(line.trim());
    }
  } else if (!process.exitCode) {
    console.log('NOTE  shadow log 未写入 LaunchAgent 日志，但 intent_routed.llmShadow 已记录观测结果');
  }

  if (process.exitCode && process.exitCode !== 0) {
    console.error('\n验证未完全通过。');
  } else {
    console.log('\n验证通过：Shadow 模式已观测 LLM 路由建议，用户侧仍走 Agent fallback。');
  }
}

main().catch((err) => {
  console.error(err instanceof Error ? err.stack ?? err.message : err);
  process.exit(1);
});