feat: add orchestrator shadow observability

admin-bootstrap.mjs

@@ -20,6 +20,7 @@ import { createAssetGatewayConfigService } from './asset-gateway.mjs';
 import { createImageMakeAdminConfigService } from './image-make-admin-config.mjs';
 import { createMemoryV2AdminConfigService } from './memory-v2-admin-config.mjs';
 import { createOrchestratorAdminConfigService } from './services/orchestrator/admin-config.mjs';
+import { createOrchestratorObservabilityService } from './services/orchestrator/observability.mjs';
 import { createSkillRuntimeAdminConfigService } from './skill-runtime-admin-config.mjs';
 import { createSystemDisclosurePolicyService } from './system-disclosure-policy.mjs';
 import { createMindSearchConfigService } from './mindsearch-config.mjs';
@@ -112,6 +113,10 @@ export async function createAdminServices(env = {}) {
   const memoryV2ConfigService = createMemoryV2AdminConfigService(pool);
   const orchestratorConfigService = createOrchestratorAdminConfigService(pool);
   await orchestratorConfigService.ensureSchema();
+  const orchestratorObservabilityService = createOrchestratorObservabilityService({
+    pool,
+    configService: orchestratorConfigService,
+  });
   const mindSearchConfigService = createMindSearchConfigService(pool);
   await mindSearchConfigService.ensureSchema();
   const skillRuntimeConfigService = createSkillRuntimeAdminConfigService(pool, { h5Root });
@@ -159,6 +164,7 @@ export async function createAdminServices(env = {}) {
     imageMakeAdminConfigService,
     memoryV2ConfigService,
     orchestratorConfigService,
+    orchestratorObservabilityService,
     mindSearchConfigService,
     skillRuntimeConfigService,
     systemDisclosurePolicyService,

admin-routes.mjs

@@ -32,6 +32,7 @@ function plazaRouteError(res, req, error) {
  * @param {object|null} deps.llmProviderService
  * @param {object|null} deps.memoryV2ConfigService
  * @param {object|null} deps.orchestratorConfigService
+ * @param {object|null} deps.orchestratorObservabilityService
  * @param {object|null} deps.skillRuntimeConfigService
  * @param {object|null} deps.systemDisclosurePolicyService
  * @param {object|null} deps.adminSystemTestService
@@ -49,6 +50,7 @@ export function createAdminApi({
   imageMakeAdminConfigService,
   memoryV2ConfigService,
   orchestratorConfigService,
+  orchestratorObservabilityService,
   mindSearchConfigService,
   skillRuntimeConfigService,
   systemDisclosurePolicyService,
@@ -255,6 +257,26 @@ export function createAdminApi({
     return res.json(await orchestratorConfigService.getRuntimeState({ probe: true }));
   });
 
+  adminApi.get('/orchestrator/shadow-runs', requireAdmin, async (req, res) => {
+    if (!orchestratorObservabilityService?.listShadowRuns) {
+      return res.status(503).json({ message: 'Orchestrator 观测服务未启用' });
+    }
+    return res.json(await orchestratorObservabilityService.listShadowRuns({
+      hours: req.query.hours,
+      limit: req.query.limit,
+      status: req.query.status,
+    }));
+  });
+
+  adminApi.get('/orchestrator/shadow-runs/:runId', requireAdmin, async (req, res) => {
+    if (!orchestratorObservabilityService?.getShadowRun) {
+      return res.status(503).json({ message: 'Orchestrator 观测服务未启用' });
+    }
+    const result = await orchestratorObservabilityService.getShadowRun(req.params.runId);
+    if (!result) return res.status(404).json({ message: 'Shadow run 不存在' });
+    return res.json(result);
+  });
+
   adminApi.get('/skill-runtime/config', requireAdmin, async (_req, res) => {
     if (!skillRuntimeConfigService?.getAdminConfig) {
       return res.status(503).json({ message: 'Skill Runtime 配置服务未启用' });

admin-routes.test.mjs

@@ -138,6 +138,22 @@ test('admin orchestrator routes expose a versioned plug-in control plane', async
         };
       },
     },
+    orchestratorObservabilityService: {
+      async listShadowRuns(options) {
+        assert.deepEqual(options, { hours: '12', limit: '25', status: 'failed' });
+        return {
+          metrics: { observations: 1, successes: 0, failures: 1 },
+          runs: [{ runId: 'run-shadow-1', shadowStatus: 'failed' }],
+        };
+      },
+      async getShadowRun(runId) {
+        assert.equal(runId, 'run-shadow-1');
+        return {
+          native: { runId, status: 'succeeded' },
+          remote: { available: true, events: [{ type: 'workflow_validated' }] },
+        };
+      },
+    },
     plazaPosts: null,
     plazaOps: null,
     wechatAdmin: null,
@@ -174,6 +190,20 @@ test('admin orchestrator routes expose a versioned plug-in control plane', async
     const runtimeBody = await runtimeResponse.json();
     assert.equal(runtimeBody.source, 'admin-db');
     assert.equal(runtimeBody.serviceHealth.status, 'healthy');
+
+    const shadowRunsResponse = await fetch(
+      `${server.baseUrl}/admin-api/orchestrator/shadow-runs?hours=12&limit=25&status=failed`,
+      { headers: { cookie: 'h5_user_session=token-admin' } },
+    );
+    assert.equal(shadowRunsResponse.status, 200);
+    assert.equal((await shadowRunsResponse.json()).metrics.failures, 1);
+
+    const shadowRunResponse = await fetch(
+      `${server.baseUrl}/admin-api/orchestrator/shadow-runs/run-shadow-1`,
+      { headers: { cookie: 'h5_user_session=token-admin' } },
+    );
+    assert.equal(shadowRunResponse.status, 200);
+    assert.equal((await shadowRunResponse.json()).remote.available, true);
   } finally {
     await server.close();
   }

admin-server.mjs

@@ -80,6 +80,7 @@ const CONSOLES = {
         imageMakeAdminConfigService: services.imageMakeAdminConfigService,
         memoryV2ConfigService: services.memoryV2ConfigService,
         orchestratorConfigService: services.orchestratorConfigService,
+        orchestratorObservabilityService: services.orchestratorObservabilityService,
         mindSearchConfigService: services.mindSearchConfigService,
         systemDisclosurePolicyService: services.systemDisclosurePolicyService,
         adminSystemTestService: services.adminSystemTestService,

agent-run-gateway.mjs

@@ -342,6 +342,7 @@ export function createAgentRunGateway({
   function dispatchShadowObservation(input) {
     if (typeof observeWorkflowRun !== 'function' || input.toolMode !== 'code') return;
     setImmediate(() => {
+      const observationStartedAt = nowMs();
       void Promise.resolve()
         .then(() => observeWorkflowRun(input))
         .then(async (result) => {
@@ -352,6 +353,9 @@ export function createAgentRunGateway({
             configVersion: result.configVersion ?? null,
             status: result.shadowRun?.status ?? null,
             phase: result.shadowRun?.phase ?? null,
+            taskType: result.shadowRun?.plan?.taskType ?? input.taskType ?? null,
+            executorAdapter: result.shadowRun?.plan?.executorAdapter ?? null,
+            latencyMs: Math.max(0, nowMs() - observationStartedAt),
           });
         })
         .catch(async (error) => {
@@ -362,6 +366,7 @@ export function createAgentRunGateway({
           await appendEvent(input.runId, 'workflow_shadow_failed', {
             code: String(error?.code ?? 'WORKFLOW_SHADOW_FAILED').slice(0, 128),
             message: String(error instanceof Error ? error.message : error).slice(0, 1000),
+            latencyMs: Math.max(0, nowMs() - observationStartedAt),
           }).catch((appendError) => {
             console.warn(
               '[AgentRun] workflow shadow failure event skipped:',

agent-run-gateway.test.mjs

@@ -420,13 +420,15 @@ test('code run shadow observation is fire-and-forget and records completion sepa
     (event) => event.eventType === 'workflow_shadow_completed',
   ));
   const event = pool.events.find((item) => item.eventType === 'workflow_shadow_completed');
-  assert.deepEqual(JSON.parse(event.dataJson), {
+  const eventData = JSON.parse(event.dataJson);
-    engine: 'langgraph',
+  assert.equal(eventData.engine, 'langgraph');
-    mode: 'shadow',
+  assert.equal(eventData.mode, 'shadow');
-    configVersion: 3,
+  assert.equal(eventData.configVersion, 3);
-    status: 'succeeded',
+  assert.equal(eventData.status, 'succeeded');
-    phase: 'completed',
+  assert.equal(eventData.phase, 'completed');
-  });
+  assert.equal(eventData.taskType, 'repo_refactor');
+  assert.equal(eventData.executorAdapter, null);
+  assert.equal(Number.isFinite(eventData.latencyMs), true);
 });
 
 test('shadow observer failure cannot fail a Native run and chat runs are not observed', async () => {

db.mjs

@@ -351,9 +351,16 @@ export async function migrateSchema(pool) {
       data_json JSON NULL,
       created_at BIGINT NOT NULL,
       KEY idx_h5_agent_run_event_run (run_id, created_at),
+      KEY idx_h5_agent_run_event_type_time (event_type, created_at),
       CONSTRAINT fk_h5_agent_run_event_run FOREIGN KEY (run_id) REFERENCES h5_agent_runs(id) ON DELETE CASCADE
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
   `);
+  if (!(await indexExists(pool, 'h5_agent_run_events', 'idx_h5_agent_run_event_type_time'))) {
+    await pool.query(
+      `ALTER TABLE h5_agent_run_events
+       ADD KEY idx_h5_agent_run_event_type_time (event_type, created_at)`,
+    );
+  }
 
   // MindSpace conversation packages: additive provenance tables for grouping
   // images, files, pages, and publications created during a chat session.

deploy/orchestrator/compose.yaml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: postgres:17-alpine
+    image: postgres:16-alpine
     restart: unless-stopped
     environment:
       POSTGRES_DB: memind_orchestrator
@@ -17,6 +17,7 @@ services:
       - orchestrator_internal
 
   orchestrator:
+    image: memind/workflow-orchestrator:local
     build:
       context: ../../services/orchestrator
       dockerfile: Dockerfile

docs/architecture/memind-orchestrator-boundary.md

@@ -105,6 +105,11 @@ failure:
 Successful observations are projected as `workflow_shadow_completed`; graph
 checkpoints stay in the Orchestrator-owned PostgreSQL database.
 
+Both terminal projection events contain bounded observation latency. The
+Memind-owned observability service aggregates those events and may join them to
+the product run status. Only per-run detail calls cross the service boundary to
+read LangGraph checkpoint state and node events.
+
 ## Deployment evolution
 
 1. Local native Orchestrator process with an explicit MemorySaver for debugging.

ops/src/api/admin.ts

@@ -257,6 +257,85 @@ export type OrchestratorRuntimeState = OrchestratorConfigState & {
   serviceHealth: OrchestratorServiceHealth;
 };
 
+export type OrchestratorShadowMetrics = {
+  observations: number;
+  successes: number;
+  failures: number;
+  successRate: number | null;
+  failureRate: number | null;
+  latencyP50Ms: number | null;
+  latencyP95Ms: number | null;
+  nativeSucceeded: number;
+  nativeFailed: number;
+  lastObservedAt: number | null;
+  sampled: boolean;
+};
+
+export type OrchestratorShadowRun = {
+  eventId: string;
+  runId: string;
+  requestId: string;
+  userId: string;
+  sessionId: string | null;
+  nativeStatus: string;
+  nativeAttempts: number;
+  shadowStatus: 'succeeded' | 'failed';
+  engine: string;
+  configVersion: number | null;
+  phase: string | null;
+  taskType: string | null;
+  executorAdapter: string | null;
+  latencyMs: number | null;
+  error: { code: string; message: string } | null;
+  observedAt: number;
+  nativeCompletedAt: number | null;
+};
+
+export type OrchestratorShadowRunList = {
+  generatedAt: number;
+  window: { hours: number; from: number };
+  metrics: OrchestratorShadowMetrics;
+  runs: OrchestratorShadowRun[];
+};
+
+export type OrchestratorShadowRunDetail = {
+  native: {
+    runId: string;
+    requestId: string;
+    userId: string;
+    sessionId: string | null;
+    status: string;
+    attempts: number;
+    error: string | null;
+    createdAt: number;
+    updatedAt: number;
+    startedAt: number | null;
+    completedAt: number | null;
+  };
+  localEvents: Array<{
+    eventId: string;
+    type: string;
+    data: Record<string, unknown> | null;
+    createdAt: number;
+  }>;
+  remote: {
+    available: boolean;
+    state: {
+      status?: string;
+      phase?: string;
+      plan?: Record<string, unknown>;
+      result?: Record<string, unknown>;
+    } | null;
+    events: Array<{
+      sequence?: number;
+      type?: string;
+      timestamp?: number;
+      data?: Record<string, unknown> | null;
+    }>;
+    error: { code: string; message: string } | null;
+  };
+};
+
 // ─── Summary ──────────────────────────────────────────────────────────────────
 
 export async function fetchAdminSummary() {
@@ -293,6 +372,26 @@ export async function fetchOrchestratorRuntime() {
   return adminFetch<OrchestratorRuntimeState>('/admin-api/orchestrator/runtime');
 }
 
+export async function fetchOrchestratorShadowRuns(params: {
+  hours?: number;
+  limit?: number;
+  status?: 'all' | 'succeeded' | 'failed';
+} = {}) {
+  const query = new URLSearchParams();
+  if (params.hours) query.set('hours', String(params.hours));
+  if (params.limit) query.set('limit', String(params.limit));
+  if (params.status) query.set('status', params.status);
+  return adminFetch<OrchestratorShadowRunList>(
+    `/admin-api/orchestrator/shadow-runs?${query}`,
+  );
+}
+
+export async function fetchOrchestratorShadowRun(runId: string) {
+  return adminFetch<OrchestratorShadowRunDetail>(
+    `/admin-api/orchestrator/shadow-runs/${encodeURIComponent(runId)}`,
+  );
+}
+
 // ─── Users ────────────────────────────────────────────────────────────────────
 
 export async function fetchAdminUsers(params: {

ops/src/pages/admin/OrchestratorPage.tsx

@@ -7,6 +7,7 @@ import {
   type OrchestratorConfigState,
   type OrchestratorServiceHealth,
 } from '../../api/admin';
+import { OrchestratorShadowPanel } from './OrchestratorShadowPanel';
 
 function listToText(values: string[]) {
   return values.join('\n');
@@ -160,6 +161,8 @@ export function OrchestratorPage() {
         </p>
       </div>
 
+      <OrchestratorShadowPanel />
+
       <div className="card grid">
         <h3 style={{ margin: 0 }}>路由模式</h3>
         <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(220px, 1fr))', gap: 16 }}>

ops/src/pages/admin/OrchestratorShadowPanel.tsx

@@ -0,0 +1,229 @@
+import { useEffect, useState } from 'react';
+import {
+  fetchOrchestratorShadowRun,
+  fetchOrchestratorShadowRuns,
+  type OrchestratorShadowRunDetail,
+  type OrchestratorShadowRunList,
+} from '../../api/admin';
+
+function formatTime(value: number | null | undefined) {
+  return value ? new Date(value).toLocaleString('zh-CN', { hour12: false }) : '—';
+}
+
+function formatPercent(value: number | null) {
+  return value == null ? '—' : `${(value * 100).toFixed(1)}%`;
+}
+
+function formatLatency(value: number | null) {
+  return value == null ? '—' : `${value} ms`;
+}
+
+function shortId(value: string) {
+  return value.length > 16 ? `${value.slice(0, 8)}…${value.slice(-6)}` : value;
+}
+
+const metricCardStyle = {
+  border: '1px solid #e4e9e6',
+  borderRadius: 10,
+  padding: 12,
+  minWidth: 140,
+} as const;
+
+export function OrchestratorShadowPanel() {
+  const [hours, setHours] = useState(24);
+  const [status, setStatus] = useState<'all' | 'succeeded' | 'failed'>('all');
+  const [data, setData] = useState<OrchestratorShadowRunList | null>(null);
+  const [selected, setSelected] = useState<OrchestratorShadowRunDetail | null>(null);
+  const [loading, setLoading] = useState(true);
+  const [detailLoading, setDetailLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  const load = async () => {
+    setLoading(true);
+    setError(null);
+    try {
+      setData(await fetchOrchestratorShadowRuns({ hours, status, limit: 100 }));
+    } catch (err) {
+      setError(err instanceof Error ? err.message : 'Shadow 指标加载失败');
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  useEffect(() => {
+    void load();
+  }, [hours, status]);
+
+  const openDetail = async (runId: string) => {
+    setDetailLoading(true);
+    setError(null);
+    try {
+      setSelected(await fetchOrchestratorShadowRun(runId));
+    } catch (err) {
+      setError(err instanceof Error ? err.message : 'Shadow run 详情加载失败');
+    } finally {
+      setDetailLoading(false);
+    }
+  };
+
+  const metrics = data?.metrics;
+
+  return (
+    <div className="card grid">
+      <div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', gap: 12, flexWrap: 'wrap' }}>
+        <div>
+          <h3 style={{ margin: 0 }}>Shadow 运行观测</h3>
+          <p style={{ margin: '4px 0 0', color: '#68716c', fontSize: 13 }}>
+            Native 执行结果与 LangGraph 观察结果并排展示，不改变任务执行权。
+          </p>
+        </div>
+        <div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
+          <select value={hours} onChange={(event) => setHours(Number(event.target.value))}>
+            <option value={1}>最近 1 小时</option>
+            <option value={24}>最近 24 小时</option>
+            <option value={168}>最近 7 天</option>
+            <option value={720}>最近 30 天</option>
+          </select>
+          <select
+            value={status}
+            onChange={(event) => setStatus(event.target.value as typeof status)}
+          >
+            <option value="all">全部结果</option>
+            <option value="succeeded">仅成功</option>
+            <option value="failed">仅失败</option>
+          </select>
+          <button type="button" className="btn secondary" onClick={() => void load()} disabled={loading}>
+            {loading ? '刷新中…' : '刷新'}
+          </button>
+        </div>
+      </div>
+
+      {error ? <p className="alert">{error}</p> : null}
+
+      <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(140px, 1fr))', gap: 10 }}>
+        <div style={metricCardStyle}>
+          <small>观察请求</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.observations ?? '—'}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>Shadow 成功率</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{formatPercent(metrics?.successRate ?? null)}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>失败数</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.failures ?? '—'}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>延迟 P50</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{formatLatency(metrics?.latencyP50Ms ?? null)}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>延迟 P95</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{formatLatency(metrics?.latencyP95Ms ?? null)}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>Native 成功 / 失败</small>
+          <div style={{ fontSize: 20, marginTop: 7 }}>
+            {metrics ? `${metrics.nativeSucceeded} / ${metrics.nativeFailed}` : '—'}
+          </div>
+        </div>
+      </div>
+
+      {metrics?.sampled ? (
+        <p className="warn">当前指标基于最近 5000 条 Shadow 结果采样。</p>
+      ) : null}
+
+      <div style={{ overflowX: 'auto' }}>
+        <table style={{ width: '100%', borderCollapse: 'collapse', minWidth: 900 }}>
+          <thead>
+            <tr style={{ textAlign: 'left', borderBottom: '1px solid #dce4df' }}>
+              <th style={{ padding: 8 }}>时间</th>
+              <th style={{ padding: 8 }}>Run</th>
+              <th style={{ padding: 8 }}>Shadow</th>
+              <th style={{ padding: 8 }}>Native</th>
+              <th style={{ padding: 8 }}>延迟</th>
+              <th style={{ padding: 8 }}>任务 / Adapter</th>
+              <th style={{ padding: 8 }}>错误</th>
+            </tr>
+          </thead>
+          <tbody>
+            {(data?.runs ?? []).map((run) => (
+              <tr key={run.eventId} style={{ borderBottom: '1px solid #edf1ef' }}>
+                <td style={{ padding: 8, whiteSpace: 'nowrap' }}>{formatTime(run.observedAt)}</td>
+                <td style={{ padding: 8 }}>
+                  <button
+                    type="button"
+                    className="btn secondary"
+                    title={run.runId}
+                    onClick={() => void openDetail(run.runId)}
+                    disabled={detailLoading}
+                  >
+                    {shortId(run.runId)}
+                  </button>
+                </td>
+                <td style={{ padding: 8, color: run.shadowStatus === 'succeeded' ? '#2f6f57' : '#a23a32' }}>
+                  {run.shadowStatus}
+                </td>
+                <td style={{ padding: 8 }}>{run.nativeStatus}</td>
+                <td style={{ padding: 8 }}>{formatLatency(run.latencyMs)}</td>
+                <td style={{ padding: 8 }}>
+                  {run.taskType ?? '—'}
+                  <small style={{ display: 'block', color: '#68716c' }}>
+                    {run.executorAdapter ?? '—'}
+                  </small>
+                </td>
+                <td style={{ padding: 8, maxWidth: 260 }}>
+                  {run.error ? `${run.error.code}: ${run.error.message}` : '—'}
+                </td>
+              </tr>
+            ))}
+            {!loading && !data?.runs.length ? (
+              <tr>
+                <td colSpan={7} style={{ padding: 16, textAlign: 'center', color: '#68716c' }}>
+                  当前时间范围内没有 Shadow 结果。
+                </td>
+              </tr>
+            ) : null}
+          </tbody>
+        </table>
+      </div>
+
+      {selected ? (
+        <div style={{ borderTop: '1px solid #dce4df', paddingTop: 16 }}>
+          <div style={{ display: 'flex', justifyContent: 'space-between', gap: 12 }}>
+            <div>
+              <h4 style={{ margin: 0 }}>Run 详情：{selected.native.runId}</h4>
+              <p style={{ margin: '6px 0', color: '#68716c' }}>
+                Native {selected.native.status} · attempts {selected.native.attempts}
+                {' · '}
+                LangGraph {selected.remote.available ? (selected.remote.state?.status ?? 'available') : 'unavailable'}
+              </p>
+            </div>
+            <button type="button" className="btn secondary" onClick={() => setSelected(null)}>
+              关闭
+            </button>
+          </div>
+          {selected.remote.error ? (
+            <p className="alert">
+              {selected.remote.error.code}: {selected.remote.error.message}
+            </p>
+          ) : null}
+          <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))', gap: 12 }}>
+            <div>
+              <strong>LangGraph checkpoint</strong>
+              <pre style={{ whiteSpace: 'pre-wrap', overflowWrap: 'anywhere', fontSize: 12 }}>
+                {JSON.stringify(selected.remote.state, null, 2)}
+              </pre>
+            </div>
+            <div>
+              <strong>LangGraph 节点事件</strong>
+              <pre style={{ whiteSpace: 'pre-wrap', overflowWrap: 'anywhere', fontSize: 12 }}>
+                {JSON.stringify(selected.remote.events, null, 2)}
+              </pre>
+            </div>
+          </div>
+        </div>
+      ) : null}
+    </div>
+  );
+}

schema.sql

@@ -422,6 +422,7 @@ CREATE TABLE IF NOT EXISTS h5_agent_run_events (
   data_json JSON NULL,
   created_at BIGINT NOT NULL,
   KEY idx_h5_agent_run_event_run (run_id, created_at),
+  KEY idx_h5_agent_run_event_type_time (event_type, created_at),
   CONSTRAINT fk_h5_agent_run_event_run FOREIGN KEY (run_id) REFERENCES h5_agent_runs(id) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 

scripts/smoke-orchestrator-shadow.mjs

@@ -0,0 +1,148 @@
+#!/usr/bin/env node
+import crypto from 'node:crypto';
+import { createAgentRunGateway } from '../agent-run-gateway.mjs';
+import { createDbPool, isDatabaseConfigured } from '../db.mjs';
+import { createOrchestratorAdminConfigService } from '../services/orchestrator/admin-config.mjs';
+import { createOrchestratorObservabilityService } from '../services/orchestrator/observability.mjs';
+import { createWorkflowShadowObserver } from '../services/orchestrator/shadow-observer.mjs';
+import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
+
+loadMemindEnvFiles(new URL('..', import.meta.url).pathname);
+
+const args = new Set(process.argv.slice(2));
+const cleanup = args.has('--cleanup');
+const restoreConfig = args.has('--restore-config');
+const serviceUrl = String(
+  process.env.MEMIND_ORCHESTRATOR_URL ?? 'http://127.0.0.1:8093',
+).trim().replace(/\/$/, '');
+const serviceToken = String(process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN ?? '').trim();
+
+function assertLocalServiceUrl(value) {
+  const url = new URL(value);
+  if (!['127.0.0.1', 'localhost', '::1'].includes(url.hostname) && !args.has('--allow-remote')) {
+    throw new Error('Shadow smoke defaults to a loopback Orchestrator; pass --allow-remote explicitly');
+  }
+}
+
+async function waitForShadowEvent(pool, runId, timeoutMs = 15_000) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const [rows] = await pool.query(
+      `SELECT event_type, data_json, created_at
+         FROM h5_agent_run_events
+        WHERE run_id = ?
+          AND event_type IN ('workflow_shadow_completed', 'workflow_shadow_failed')
+        ORDER BY created_at DESC
+        LIMIT 1`,
+      [runId],
+    );
+    if (rows[0]) return rows[0];
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  throw new Error(`Timed out waiting for Shadow projection for run ${runId}`);
+}
+
+async function main() {
+  if (!isDatabaseConfigured()) {
+    throw new Error('Shadow smoke requires the local Memind MySQL configuration');
+  }
+  if (!serviceToken) {
+    throw new Error('MEMIND_ORCHESTRATOR_SERVICE_TOKEN is required');
+  }
+  assertLocalServiceUrl(serviceUrl);
+  const ready = await fetch(`${serviceUrl}/ready`);
+  if (!ready.ok) throw new Error(`Orchestrator readiness failed with HTTP ${ready.status}`);
+
+  const pool = createDbPool();
+  const configService = createOrchestratorAdminConfigService(pool);
+  const previous = await configService.getAdminConfig();
+  let runId = null;
+  try {
+    await configService.updateAdminConfig({
+      ...previous.config,
+      mode: 'shadow',
+      serviceUrl,
+      primaryEngine: 'langgraph',
+      workflowAllowlist: ['code-run-v1'],
+    });
+    const [users] = await pool.query(
+      `SELECT id
+         FROM h5_users
+        ORDER BY created_at ASC
+        LIMIT 1`,
+    );
+    if (!users[0]?.id) throw new Error('Shadow smoke requires at least one local user');
+
+    const observer = createWorkflowShadowObserver({
+      configService,
+      serviceToken,
+    });
+    const gateway = createAgentRunGateway({
+      pool,
+      tkmindProxy: {},
+      autoDispatch: false,
+      observeWorkflowRun: observer,
+    });
+    const requestId = `orchestrator-shadow-smoke-${Date.now()}-${crypto.randomUUID()}`;
+    const run = await gateway.createRun(users[0].id, {
+      requestId,
+      toolMode: 'code',
+      taskType: 'orchestrator_shadow_smoke',
+      userMessage: {
+        role: 'user',
+        content: [{ type: 'text', text: 'Observe this local smoke run without executing tools.' }],
+      },
+    });
+    runId = run.id;
+    const event = await waitForShadowEvent(pool, runId);
+    if (event.event_type !== 'workflow_shadow_completed') {
+      throw new Error(`Shadow smoke failed: ${JSON.stringify(event.data_json)}`);
+    }
+
+    const completedAt = Date.now();
+    await pool.query(
+      `UPDATE h5_agent_runs
+          SET status = 'succeeded', updated_at = ?, completed_at = ?
+        WHERE id = ? AND status = 'queued'`,
+      [completedAt, completedAt, runId],
+    );
+
+    const observability = createOrchestratorObservabilityService({
+      pool,
+      configService,
+      serviceToken,
+    });
+    const [summary, detail] = await Promise.all([
+      observability.listShadowRuns({ hours: 1, limit: 10 }),
+      observability.getShadowRun(runId),
+    ]);
+    if (!detail?.remote?.available || detail.remote.state?.status !== 'succeeded') {
+      throw new Error('Shadow checkpoint detail was not recoverable through the remote API');
+    }
+
+    console.log(JSON.stringify({
+      ok: true,
+      runId,
+      mode: 'shadow',
+      shadowEvent: event.event_type,
+      metrics: summary.metrics,
+      checkpointStatus: detail.remote.state.status,
+      graphEvents: detail.remote.events.map((item) => item.type),
+      cleanup,
+      restoreConfig,
+    }, null, 2));
+  } finally {
+    if (cleanup && runId) {
+      await pool.query('DELETE FROM h5_agent_runs WHERE id = ?', [runId]);
+    }
+    if (restoreConfig) {
+      await configService.updateAdminConfig(previous.config);
+    }
+    await pool.end();
+  }
+}
+
+main().catch((error) => {
+  console.error(error instanceof Error ? error.message : error);
+  process.exitCode = 1;
+});

services/orchestrator/README.md

@@ -48,6 +48,18 @@ Phase 2 adds:
 - live health probing from memindadm;
 - a separate Colima/Compose deployment artifact.
 
+Phase 2.5 adds a Memind-owned Shadow observability projection:
+
+- success/failure totals and rates;
+- P50/P95 observation latency;
+- Native terminal status beside the Shadow result;
+- recent failure codes and messages;
+- per-run LangGraph checkpoint and node-event inspection through the versioned
+  remote API.
+
+The projection reads `h5_agent_run_events` from the Memind control plane.
+LangGraph still has no access to Memind business tables.
+
 The default memindadm mode remains `off`. Canary and Active routing are not
 wired to the LangGraph executor in Phase 2. Native Agent Run remains the sole
 executor, including in Shadow mode.
@@ -84,6 +96,13 @@ MEMIND_ORCHESTRATOR_CHECKPOINT_MODE=memory pnpm dev:orchestrator
 The service binds to `127.0.0.1:8093` by default. Configure the same URL and
 service token in Portal, then enable `shadow` in `/ops/admin/orchestrator`.
 
+memindadm exposes the projection through:
+
+```text
+GET /admin-api/orchestrator/shadow-runs
+GET /admin-api/orchestrator/shadow-runs/:runId
+```
+
 ## Colima deployment
 
 Colima is the recommended first container host on macOS because this service and

services/orchestrator/observability.mjs

@@ -0,0 +1,254 @@
+import { createRemoteWorkflowEngine } from './engine-registry.mjs';
+
+const SHADOW_EVENT_TYPES = Object.freeze([
+  'workflow_shadow_completed',
+  'workflow_shadow_failed',
+]);
+const MAX_METRIC_EVENTS = 5000;
+
+function clampInteger(value, fallback, min, max) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(parsed)));
+}
+
+function parseJsonColumn(value, fallback = null) {
+  if (value == null || value === '') return fallback;
+  if (typeof value === 'object') return value;
+  try {
+    return JSON.parse(String(value));
+  } catch {
+    return fallback;
+  }
+}
+
+function percentile(values, ratio) {
+  if (!values.length) return null;
+  const sorted = [...values].sort((left, right) => left - right);
+  const index = Math.max(0, Math.ceil(sorted.length * ratio) - 1);
+  return sorted[index];
+}
+
+function normalizeRunId(value) {
+  const runId = String(value ?? '').trim();
+  return /^[a-zA-Z0-9_-]{1,128}$/.test(runId) ? runId : '';
+}
+
+function projectShadowEventRow(row) {
+  const data = parseJsonColumn(row.data_json, {}) ?? {};
+  const succeeded = row.event_type === 'workflow_shadow_completed';
+  const latency = Number(data.latencyMs);
+  return {
+    eventId: row.event_id,
+    runId: row.run_id,
+    requestId: row.request_id,
+    userId: row.user_id,
+    sessionId: row.agent_session_id ?? null,
+    nativeStatus: row.native_status,
+    nativeAttempts: Number(row.native_attempts ?? 0),
+    shadowStatus: succeeded ? 'succeeded' : 'failed',
+    engine: data.engine ?? 'langgraph',
+    configVersion: data.configVersion == null ? null : Number(data.configVersion),
+    phase: data.phase ?? null,
+    taskType: data.taskType ?? null,
+    executorAdapter: data.executorAdapter ?? null,
+    latencyMs: Number.isFinite(latency) && latency >= 0 ? latency : null,
+    error: succeeded ? null : {
+      code: data.code ?? 'WORKFLOW_SHADOW_FAILED',
+      message: data.message ?? 'Shadow observation failed',
+    },
+    observedAt: Number(row.event_created_at ?? 0),
+    nativeCompletedAt: row.native_completed_at == null
+      ? null
+      : Number(row.native_completed_at),
+  };
+}
+
+function summarizeRuns(runs, { capped = false } = {}) {
+  const successes = runs.filter((run) => run.shadowStatus === 'succeeded').length;
+  const failures = runs.length - successes;
+  const latencies = runs
+    .map((run) => run.latencyMs)
+    .filter((value) => Number.isFinite(value));
+  return {
+    observations: runs.length,
+    successes,
+    failures,
+    successRate: runs.length ? successes / runs.length : null,
+    failureRate: runs.length ? failures / runs.length : null,
+    latencyP50Ms: percentile(latencies, 0.5),
+    latencyP95Ms: percentile(latencies, 0.95),
+    nativeSucceeded: runs.filter((run) => run.nativeStatus === 'succeeded').length,
+    nativeFailed: runs.filter((run) => run.nativeStatus === 'failed').length,
+    lastObservedAt: runs[0]?.observedAt ?? null,
+    sampled: capped,
+  };
+}
+
+function safeRemoteError(error) {
+  return {
+    code: String(error?.code ?? 'ORCHESTRATOR_UNAVAILABLE').slice(0, 128),
+    message: String(error instanceof Error ? error.message : error).slice(0, 1000),
+  };
+}
+
+export function createOrchestratorObservabilityService({
+  pool,
+  configService,
+  serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  if (!pool?.query) throw new Error('Orchestrator observability requires a database pool');
+  if (!configService?.getRuntimeState) {
+    throw new Error('Orchestrator observability requires config service');
+  }
+
+  async function loadShadowEvents({ hours = 24 } = {}) {
+    const normalizedHours = clampInteger(hours, 24, 1, 24 * 30);
+    const from = Date.now() - normalizedHours * 60 * 60 * 1000;
+    const [rows] = await pool.query(
+      `SELECT
+         e.id AS event_id,
+         e.run_id,
+         e.event_type,
+         e.data_json,
+         e.created_at AS event_created_at,
+         r.request_id,
+         r.user_id,
+         r.agent_session_id,
+         r.status AS native_status,
+         r.attempts AS native_attempts,
+         r.completed_at AS native_completed_at
+       FROM h5_agent_run_events e
+       INNER JOIN h5_agent_runs r ON r.id = e.run_id
+       WHERE e.event_type IN (?, ?)
+         AND e.created_at >= ?
+       ORDER BY e.created_at DESC, e.id DESC
+       LIMIT ?`,
+      [...SHADOW_EVENT_TYPES, from, MAX_METRIC_EVENTS],
+    );
+    return {
+      from,
+      hours: normalizedHours,
+      capped: rows.length >= MAX_METRIC_EVENTS,
+      runs: rows.map(projectShadowEventRow),
+    };
+  }
+
+  return {
+    async listShadowRuns({
+      hours = 24,
+      limit = 50,
+      status = 'all',
+    } = {}) {
+      const loaded = await loadShadowEvents({ hours });
+      const normalizedLimit = clampInteger(limit, 50, 1, 200);
+      const normalizedStatus = ['succeeded', 'failed'].includes(status) ? status : 'all';
+      const filtered = normalizedStatus === 'all'
+        ? loaded.runs
+        : loaded.runs.filter((run) => run.shadowStatus === normalizedStatus);
+      return {
+        generatedAt: Date.now(),
+        window: {
+          hours: loaded.hours,
+          from: loaded.from,
+        },
+        metrics: summarizeRuns(loaded.runs, { capped: loaded.capped }),
+        runs: filtered.slice(0, normalizedLimit),
+      };
+    },
+
+    async getShadowRun(runId) {
+      const normalizedRunId = normalizeRunId(runId);
+      if (!normalizedRunId) return null;
+      const [runRows] = await pool.query(
+        `SELECT id, request_id, user_id, agent_session_id, status, attempts,
+                error_message, created_at, updated_at, started_at, completed_at
+           FROM h5_agent_runs
+          WHERE id = ?
+          LIMIT 1`,
+        [normalizedRunId],
+      );
+      if (!runRows[0]) return null;
+      const [eventRows] = await pool.query(
+        `SELECT id AS event_id, event_type, data_json, created_at
+           FROM h5_agent_run_events
+          WHERE run_id = ? AND event_type IN (?, ?)
+          ORDER BY created_at ASC, id ASC`,
+        [normalizedRunId, ...SHADOW_EVENT_TYPES],
+      );
+      const native = runRows[0];
+      const localEvents = eventRows.map((row) => ({
+        eventId: row.event_id,
+        type: row.event_type,
+        data: parseJsonColumn(row.data_json, null),
+        createdAt: Number(row.created_at),
+      }));
+
+      let remote = {
+        available: false,
+        state: null,
+        events: [],
+        error: null,
+      };
+      try {
+        const runtimeState = await configService.getRuntimeState();
+        if (runtimeState.config.serviceUrl) {
+          const engine = createRemoteWorkflowEngine({
+            baseUrl: runtimeState.config.serviceUrl,
+            serviceToken,
+            timeoutMs: runtimeState.config.requestTimeoutMs,
+            fetchImpl,
+          });
+          const [state, events] = await Promise.all([
+            engine.getState(normalizedRunId),
+            (async () => {
+              const collected = [];
+              for await (const event of engine.streamEvents(normalizedRunId)) {
+                collected.push(event);
+                if (collected.length >= 500) break;
+              }
+              return collected;
+            })(),
+          ]);
+          remote = {
+            available: true,
+            state,
+            events,
+            error: null,
+          };
+        }
+      } catch (error) {
+        remote.error = safeRemoteError(error);
+      }
+
+      return {
+        native: {
+          runId: native.id,
+          requestId: native.request_id,
+          userId: native.user_id,
+          sessionId: native.agent_session_id ?? null,
+          status: native.status,
+          attempts: Number(native.attempts ?? 0),
+          error: native.error_message ?? null,
+          createdAt: Number(native.created_at ?? 0),
+          updatedAt: Number(native.updated_at ?? 0),
+          startedAt: native.started_at == null ? null : Number(native.started_at),
+          completedAt: native.completed_at == null ? null : Number(native.completed_at),
+        },
+        localEvents,
+        remote,
+      };
+    },
+  };
+}
+
+export const orchestratorObservabilityInternals = {
+  MAX_METRIC_EVENTS,
+  SHADOW_EVENT_TYPES,
+  normalizeRunId,
+  parseJsonColumn,
+  percentile,
+  projectShadowEventRow,
+  summarizeRuns,
+};

services/orchestrator/observability.test.mjs

@@ -0,0 +1,150 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createOrchestratorObservabilityService } from './observability.mjs';
+
+function response(body, status = 200) {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { 'content-type': 'application/json' },
+  });
+}
+
+test('observability aggregates shadow outcomes and latency percentiles', async () => {
+  const rows = [
+    {
+      event_id: 'event-3',
+      run_id: 'run-3',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 300, engine: 'langgraph', taskType: 'refactor' },
+      event_created_at: 3000,
+      request_id: 'request-3',
+      user_id: 'user-3',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 3100,
+    },
+    {
+      event_id: 'event-2',
+      run_id: 'run-2',
+      event_type: 'workflow_shadow_failed',
+      data_json: JSON.stringify({ latencyMs: 200, code: 'TIMEOUT', message: 'timed out' }),
+      event_created_at: 2000,
+      request_id: 'request-2',
+      user_id: 'user-2',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 2200,
+    },
+    {
+      event_id: 'event-1',
+      run_id: 'run-1',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 100, executorAdapter: 'native-agent-run' },
+      event_created_at: 1000,
+      request_id: 'request-1',
+      user_id: 'user-1',
+      native_status: 'failed',
+      native_attempts: 2,
+      native_completed_at: 1500,
+    },
+  ];
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query(sql) {
+        assert.match(sql, /INNER JOIN h5_agent_runs/);
+        return [rows];
+      },
+    },
+    configService: { getRuntimeState() {} },
+  });
+
+  const result = await service.listShadowRuns({ hours: 12, limit: 2 });
+  assert.equal(result.window.hours, 12);
+  assert.equal(result.metrics.observations, 3);
+  assert.equal(result.metrics.successes, 2);
+  assert.equal(result.metrics.failures, 1);
+  assert.equal(result.metrics.latencyP50Ms, 200);
+  assert.equal(result.metrics.latencyP95Ms, 300);
+  assert.equal(result.metrics.nativeSucceeded, 2);
+  assert.equal(result.runs.length, 2);
+  assert.equal(result.runs[1].error.code, 'TIMEOUT');
+});
+
+test('observability detail joins Native state with remote LangGraph checkpoint events', async () => {
+  const queries = [];
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query(sql, params) {
+        queries.push({ sql, params });
+        if (sql.includes('FROM h5_agent_runs')) {
+          return [[{
+            id: 'run-detail',
+            request_id: 'request-detail',
+            user_id: 'user-detail',
+            agent_session_id: 'session-detail',
+            status: 'succeeded',
+            attempts: 1,
+            error_message: null,
+            created_at: 100,
+            updated_at: 300,
+            started_at: 150,
+            completed_at: 300,
+          }]];
+        }
+        return [[{
+          event_id: 'local-event',
+          event_type: 'workflow_shadow_completed',
+          data_json: { latencyMs: 25 },
+          created_at: 250,
+        }]];
+      },
+    },
+    configService: {
+      async getRuntimeState() {
+        return {
+          config: {
+            serviceUrl: 'http://orchestrator.internal:8093',
+            requestTimeoutMs: 1000,
+          },
+        };
+      },
+    },
+    serviceToken: 'detail-token',
+    fetchImpl: async (url, init) => {
+      assert.equal(init.headers.authorization, 'Bearer detail-token');
+      if (String(url).endsWith('/events')) {
+        return response({
+          events: [{ sequence: 1, type: 'workflow_validated' }],
+        });
+      }
+      return response({
+        runId: 'run-detail',
+        status: 'succeeded',
+        plan: { executorAdapter: 'native-agent-run' },
+      });
+    },
+  });
+
+  const detail = await service.getShadowRun('run-detail');
+  assert.equal(detail.native.status, 'succeeded');
+  assert.equal(detail.localEvents[0].data.latencyMs, 25);
+  assert.equal(detail.remote.available, true);
+  assert.equal(detail.remote.state.plan.executorAdapter, 'native-agent-run');
+  assert.equal(detail.remote.events[0].type, 'workflow_validated');
+  assert.equal(queries.length, 2);
+});
+
+test('observability rejects unsafe run ids before querying', async () => {
+  let queries = 0;
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        queries += 1;
+        return [[]];
+      },
+    },
+    configService: { getRuntimeState() {} },
+  });
+  assert.equal(await service.getShadowRun('../unsafe'), null);
+  assert.equal(queries, 0);
+});