chore: add runtime observability ops

scripts/build-portal-runtime.mjs

@@ -267,6 +267,14 @@ async function writeMetadata() {
     path.join(root, 'scripts', 'wechat-mp-menu.mjs'),
     path.join(runtimeRoot, 'scripts', 'wechat-mp-menu.mjs'),
   );
+  await fs.copyFile(
+    path.join(root, 'scripts', 'check-stream-runtime.mjs'),
+    path.join(runtimeRoot, 'scripts', 'check-stream-runtime.mjs'),
+  );
+  await fs.copyFile(
+    path.join(root, 'scripts', 'runtime-worker-drain.mjs'),
+    path.join(runtimeRoot, 'scripts', 'runtime-worker-drain.mjs'),
+  );
   await writeFile(
     path.join(runtimeRoot, 'RUNBOOK.txt'),
     [
@@ -300,11 +308,16 @@ async function writeMetadata() {
       '  H5_USERS_ROOT / MINDSPACE_STORAGE_ROOT / MEMIND_SHARED_PUBLISH_ROOT',
       '',
       'Deployment and operations transport:',
-      '  105 fixed IP: 120.26.184.105',
-      '  103 / Studio fixed IP: 58.38.22.103',
-      '  m.tkmind.cn: 105 nginx -> 127.0.0.1:19081 -> reverse SSH tunnel -> Portal :8081',
-      '  scripts/memind-portal-tunnel.sh must stay in runtime; release restarts cn.tkmind.memind-portal-tunnel',
-      '  Do not switch back to 10.10.* LAN paths unless explicitly required.',
+      '  H5 public domain: mm.tkmind.cn',
+      '  Current public path: mm.tkmind.cn -> local nginx -> Portal :8081',
+      '  Legacy rollback-only path: m.tkmind.cn -> 105 nginx -> reverse SSH tunnel -> Portal :8081',
+      '  Future H5 traffic must not depend on 105 forwarding unless explicitly rolling back.',
+      '',
+      'Streaming runtime operations:',
+      '  node scripts/check-stream-runtime.mjs',
+      '  node scripts/runtime-worker-drain.mjs status',
+      '  node scripts/runtime-worker-drain.mjs drain goosed-3',
+      '  node scripts/runtime-worker-drain.mjs undrain goosed-3',
       '',
     ].join('\n'),
   );
@@ -322,6 +335,8 @@ async function main() {
   await writeMetadata();
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'run-memind-portal-prod.sh'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'wechat-mp-menu.mjs'), 0o755);
+  await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-stream-runtime.mjs'), 0o755);
+  await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-drain.mjs'), 0o755);
   await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
   console.log('');
   console.log(`Portal runtime 已生成: ${runtimeRoot}`);

scripts/check-stream-runtime.mjs

@@ -0,0 +1,98 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+import { Agent, fetch as undiciFetch } from 'undici';
+
+function loadEnvFile(filePath) {
+  if (!fs.existsSync(filePath)) return;
+  for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const idx = trimmed.indexOf('=');
+    if (idx < 0) continue;
+    const key = trimmed.slice(0, idx).trim();
+    const value = trimmed.slice(idx + 1).trim();
+    if (!process.env[key]) process.env[key] = value;
+  }
+}
+
+loadEnvFile(path.join(process.cwd(), '.env'));
+
+const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
+const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, '');
+const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
+  .split(',')
+  .map((value) => value.trim())
+  .filter(Boolean);
+
+async function fetchText(url, init = {}) {
+  const res = await undiciFetch(url, {
+    ...init,
+    dispatcher: url.startsWith('https://127.0.0.1') ? insecureDispatcher : undefined,
+  });
+  const text = await res.text();
+  return { res, text };
+}
+
+async function checkJson(pathname) {
+  const { res, text } = await fetchText(`${publicBase}${pathname}`);
+  let json = null;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    json = null;
+  }
+  return { ok: res.ok, status: res.status, json, text: json ? undefined : text.slice(0, 160) };
+}
+
+async function checkSseHeaders(pathname) {
+  const { res } = await fetchText(`${publicBase}${pathname}`, {
+    headers: { Accept: 'text/event-stream' },
+  });
+  return {
+    ok: res.status === 401 || res.ok,
+    status: res.status,
+    xAccelBuffering: res.headers.get('x-accel-buffering'),
+    contentType: res.headers.get('content-type'),
+    cacheControl: res.headers.get('cache-control'),
+  };
+}
+
+async function checkTarget(target) {
+  const { res, text } = await fetchText(new URL('/status', target).toString());
+  return { target, ok: res.ok, status: res.status, text: text.slice(0, 80) };
+}
+
+const result = {
+  ok: true,
+  publicBase,
+  checkedAt: new Date().toISOString(),
+  status: await checkJson('/api/status').catch((err) => ({ ok: false, error: err.message })),
+  runtime: await checkJson('/api/runtime/status').catch((err) => ({ ok: false, error: err.message })),
+  sse: {
+    sessions: await checkSseHeaders('/api/sessions/check-stream-runtime/events').catch((err) => ({
+      ok: false,
+      error: err.message,
+    })),
+    agentRuns: await checkSseHeaders('/api/agent/runs/check-stream-runtime/events').catch((err) => ({
+      ok: false,
+      error: err.message,
+    })),
+  },
+  targets: await Promise.all(targets.map((target) => checkTarget(target).catch((err) => ({
+    target,
+    ok: false,
+    error: err.message,
+  })))),
+};
+
+result.ok = Boolean(
+  result.status.ok &&
+  result.runtime.ok &&
+  result.sse.sessions.ok &&
+  result.sse.agentRuns.ok &&
+  result.targets.every((target) => target.ok),
+);
+
+console.log(JSON.stringify(result, null, 2));
+process.exit(result.ok ? 0 : 1);

scripts/runtime-worker-drain.mjs

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import path from 'node:path';
+import { createClient } from 'redis';
+
+function loadEnvFile(filePath) {
+  if (!fs.existsSync(filePath)) return;
+  for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const idx = trimmed.indexOf('=');
+    if (idx < 0) continue;
+    const key = trimmed.slice(0, idx).trim();
+    const value = trimmed.slice(idx + 1).trim();
+    if (!process.env[key]) process.env[key] = value;
+  }
+}
+
+loadEnvFile(path.join(process.cwd(), '.env'));
+
+const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
+const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
+const configuredWorkers = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
+  .split(',')
+  .map((value) => value.trim())
+  .filter(Boolean)
+  .map((_, index) => `goosed-${index + 1}`);
+const action = process.argv[2] || 'status';
+const workerId = process.argv[3] || null;
+
+function usage() {
+  console.error('Usage: node scripts/runtime-worker-drain.mjs <status|drain|undrain> [goosed-N]');
+}
+
+function workerKey(id, field) {
+  return [namespace, 'worker', id, field].join(':');
+}
+
+if (!['status', 'drain', 'undrain'].includes(action)) {
+  usage();
+  process.exit(2);
+}
+if (['drain', 'undrain'].includes(action) && !workerId) {
+  usage();
+  process.exit(2);
+}
+
+const client = createClient({ url: redisUrl });
+client.on('error', (err) => {
+  console.error(`Redis error: ${err instanceof Error ? err.message : err}`);
+});
+await client.connect();
+
+if (action === 'drain') {
+  await client.set(workerKey(workerId, 'drain'), '1');
+}
+if (action === 'undrain') {
+  await client.del(workerKey(workerId, 'drain'));
+}
+
+const keys = await client.keys(workerKey('*', 'active_streams'));
+const workers = [
+  ...configuredWorkers,
+  ...keys
+  .map((key) => key.split(':').at(-2))
+  .filter(Boolean),
+];
+if (workerId && !workers.includes(workerId)) workers.push(workerId);
+
+const rows = [];
+for (const id of [...new Set(workers)].sort()) {
+  const values = await client.mGet([
+    workerKey(id, 'active_streams'),
+    workerKey(id, 'drain'),
+    workerKey(id, 'stream_open_count'),
+    workerKey(id, 'stream_abort_count'),
+    workerKey(id, 'stream_error_count'),
+    workerKey(id, 'last_stream_started_at'),
+    workerKey(id, 'last_stream_ended_at'),
+  ]);
+  rows.push({
+    id,
+    activeStreams: Number(values[0] || 0),
+    drain: /^(1|true|yes)$/i.test(String(values[1] || '')),
+    streamOpenCount: Number(values[2] || 0),
+    streamAbortCount: Number(values[3] || 0),
+    streamErrorCount: Number(values[4] || 0),
+    lastStreamStartedAt: values[5] ? Number(values[5]) : null,
+    lastStreamEndedAt: values[6] ? Number(values[6]) : null,
+  });
+}
+
+await client.quit();
+
+console.log(JSON.stringify({
+  ok: true,
+  action,
+  workerId,
+  namespace,
+  workers: rows,
+}, null, 2));