feat: add workflow dry-run observability

2026-07-24 22:44:26 +08:00
parent c9ad841543
commit 19706eb3a0
15 changed files with 725 additions and 41 deletions
@@ -306,6 +306,52 @@ export type OrchestratorShadowRunList = {
  runs: OrchestratorShadowRun[];
 };

+export type OrchestratorExecutionPlan = {
+  eventId: string;
+  runId: string;
+  requestId: string;
+  userId: string;
+  sessionId: string | null;
+  nativeStatus: string;
+  nativeAttempts: number;
+  mode: string | null;
+  candidateEngine: string;
+  effectiveEngine: string;
+  fallbackEngine: string;
+  reason: string | null;
+  candidateReason: string | null;
+  configVersion: number | null;
+  bucket: number | null;
+  taskType: string | null;
+  dryRun: boolean;
+  handoffAllowed: boolean;
+  plannedAt: number;
+  nativeCompletedAt: number | null;
+};
+
+export type OrchestratorExecutionPlanMetrics = {
+  decisions: number;
+  candidateSelections: number;
+  candidateSelectionRate: number | null;
+  nativeSelections: number;
+  nativeSucceeded: number;
+  nativeFailed: number;
+  nativeSettledRate: number | null;
+  handoffAllowed: number;
+  distinctSessions: number;
+  lastPlannedAt: number | null;
+  candidateReasons: Array<{ value: string; count: number }>;
+  taskTypes: Array<{ value: string; count: number }>;
+  sampled: boolean;
+};
+
+export type OrchestratorExecutionPlanList = {
+  generatedAt: number;
+  window: { hours: number; from: number };
+  metrics: OrchestratorExecutionPlanMetrics;
+  plans: OrchestratorExecutionPlan[];
+};
+
 export type OrchestratorCanaryReadinessCheck = {
  id: string;
  passed: boolean;
@@ -443,6 +489,20 @@ export async function fetchOrchestratorShadowRuns(params: {
  );
 }

+export async function fetchOrchestratorExecutionPlans(params: {
+  hours?: number;
+  limit?: number;
+  selection?: 'all' | 'candidate' | 'native';
+} = {}) {
+  const query = new URLSearchParams();
+  if (params.hours) query.set('hours', String(params.hours));
+  if (params.limit) query.set('limit', String(params.limit));
+  if (params.selection) query.set('selection', params.selection);
+  return adminFetch<OrchestratorExecutionPlanList>(
+    `/admin-api/orchestrator/execution-plans?${query}`,
+  );
+}
+
 export async function fetchOrchestratorCanaryReadiness() {
  return adminFetch<OrchestratorCanaryReadiness>(
    '/admin-api/orchestrator/canary-readiness',
@@ -0,0 +1,205 @@
+import { useEffect, useState } from 'react';
+import {
+  fetchOrchestratorExecutionPlans,
+  type OrchestratorExecutionPlanList,
+} from '../../api/admin';
+
+function formatTime(value: number | null | undefined) {
+  return value ? new Date(value).toLocaleString('zh-CN', { hour12: false }) : '—';
+}
+
+function formatPercent(value: number | null) {
+  return value == null ? '—' : `${(value * 100).toFixed(1)}%`;
+}
+
+function shortId(value: string) {
+  return value.length > 16 ? `${value.slice(0, 8)}…${value.slice(-6)}` : value;
+}
+
+const metricCardStyle = {
+  border: '1px solid #e4e9e6',
+  borderRadius: 10,
+  padding: 12,
+  minWidth: 140,
+} as const;
+
+export function OrchestratorExecutionPlanPanel() {
+  const [hours, setHours] = useState(24);
+  const [selection, setSelection] = useState<'all' | 'candidate' | 'native'>('all');
+  const [data, setData] = useState<OrchestratorExecutionPlanList | null>(null);
+  const [loading, setLoading] = useState(true);
+  const [error, setError] = useState<string | null>(null);
+
+  const load = async () => {
+    setLoading(true);
+    setError(null);
+    try {
+      setData(await fetchOrchestratorExecutionPlans({
+        hours,
+        selection,
+        limit: 100,
+      }));
+    } catch (err) {
+      setError(err instanceof Error ? err.message : 'Dry-run 决策加载失败');
+    } finally {
+      setLoading(false);
+    }
+  };
+
+  useEffect(() => {
+    void load();
+  }, [hours, selection]);
+
+  const metrics = data?.metrics;
+
+  return (
+    <div className="card grid">
+      <div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', gap: 12, flexWrap: 'wrap' }}>
+        <div>
+          <h3 style={{ margin: 0 }}>执行路由 Dry-run</h3>
+          <p style={{ margin: '4px 0 0', color: '#68716c', fontSize: 13 }}>
+            Shadow 同时模拟 Canary 候选路由；所有记录的实际执行引擎仍必须是 Native。
+          </p>
+        </div>
+        <div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
+          <select value={hours} onChange={(event) => setHours(Number(event.target.value))}>
+            <option value={1}>最近 1 小时</option>
+            <option value={24}>最近 24 小时</option>
+            <option value={168}>最近 7 天</option>
+            <option value={720}>最近 30 天</option>
+          </select>
+          <select
+            value={selection}
+            onChange={(event) => setSelection(event.target.value as typeof selection)}
+          >
+            <option value="all">全部决策</option>
+            <option value="candidate">命中 LangGraph 候选</option>
+            <option value="native">未命中，保持 Native</option>
+          </select>
+          <button type="button" className="btn secondary" onClick={() => void load()} disabled={loading}>
+            {loading ? '刷新中…' : '刷新'}
+          </button>
+        </div>
+      </div>
+
+      {error ? <p className="alert">{error}</p> : null}
+      {(metrics?.handoffAllowed ?? 0) > 0 ? (
+        <p className="alert">
+          检测到 {metrics?.handoffAllowed} 条 handoffAllowed=true 记录，请立即检查执行闸门。
+        </p>
+      ) : null}
+
+      <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(140px, 1fr))', gap: 10 }}>
+        <div style={metricCardStyle}>
+          <small>路由决策</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.decisions ?? '—'}</div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>候选命中率</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>
+            {formatPercent(metrics?.candidateSelectionRate ?? null)}
+          </div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>LangGraph / Native 候选</small>
+          <div style={{ fontSize: 20, marginTop: 7 }}>
+            {metrics ? `${metrics.candidateSelections} / ${metrics.nativeSelections}` : '—'}
+          </div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>Native 终态覆盖率</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>
+            {formatPercent(metrics?.nativeSettledRate ?? null)}
+          </div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>Native 成功 / 失败</small>
+          <div style={{ fontSize: 20, marginTop: 7 }}>
+            {metrics ? `${metrics.nativeSucceeded} / ${metrics.nativeFailed}` : '—'}
+          </div>
+        </div>
+        <div style={metricCardStyle}>
+          <small>真实会话</small>
+          <div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.distinctSessions ?? '—'}</div>
+        </div>
+      </div>
+
+      {metrics ? (
+        <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(280px, 1fr))', gap: 12 }}>
+          <div>
+            <strong>选择原因</strong>
+            <p style={{ margin: '6px 0 0', color: '#68716c', fontSize: 13 }}>
+              {metrics.candidateReasons.length
+                ? metrics.candidateReasons.map((item) => `${item.value} × ${item.count}`).join('，')
+                : '暂无数据'}
+            </p>
+          </div>
+          <div>
+            <strong>任务类型</strong>
+            <p style={{ margin: '6px 0 0', color: '#68716c', fontSize: 13 }}>
+              {metrics.taskTypes.length
+                ? metrics.taskTypes.map((item) => `${item.value} × ${item.count}`).join('，')
+                : '暂无数据'}
+            </p>
+          </div>
+        </div>
+      ) : null}
+
+      {metrics?.sampled ? (
+        <p className="warn">当前指标基于最近 5000 条 Dry-run 决策采样。</p>
+      ) : null}
+
+      <div style={{ overflowX: 'auto' }}>
+        <table style={{ width: '100%', borderCollapse: 'collapse', minWidth: 980 }}>
+          <thead>
+            <tr style={{ textAlign: 'left', borderBottom: '1px solid #dce4df' }}>
+              <th style={{ padding: 8 }}>时间</th>
+              <th style={{ padding: 8 }}>Run</th>
+              <th style={{ padding: 8 }}>模式</th>
+              <th style={{ padding: 8 }}>候选 → 实际</th>
+              <th style={{ padding: 8 }}>原因 / Bucket</th>
+              <th style={{ padding: 8 }}>任务类型</th>
+              <th style={{ padding: 8 }}>Native</th>
+            </tr>
+          </thead>
+          <tbody>
+            {(data?.plans ?? []).map((plan) => (
+              <tr key={plan.eventId} style={{ borderBottom: '1px solid #edf1ef' }}>
+                <td style={{ padding: 8, whiteSpace: 'nowrap' }}>{formatTime(plan.plannedAt)}</td>
+                <td style={{ padding: 8 }} title={plan.runId}>{shortId(plan.runId)}</td>
+                <td style={{ padding: 8 }}>{plan.mode ?? '—'}</td>
+                <td style={{ padding: 8 }}>
+                  <strong style={{ color: plan.candidateEngine === 'native' ? '#68716c' : '#8a5a16' }}>
+                    {plan.candidateEngine}
+                  </strong>
+                  {' → '}
+                  {plan.effectiveEngine}
+                </td>
+                <td style={{ padding: 8 }}>
+                  {plan.candidateReason ?? plan.reason ?? '—'}
+                  <small style={{ display: 'block', color: '#68716c' }}>
+                    bucket {plan.bucket ?? '—'} · config v{plan.configVersion ?? '—'}
+                  </small>
+                </td>
+                <td style={{ padding: 8 }}>{plan.taskType ?? '—'}</td>
+                <td style={{ padding: 8 }}>
+                  {plan.nativeStatus}
+                  <small style={{ display: 'block', color: '#68716c' }}>
+                    attempts {plan.nativeAttempts}
+                  </small>
+                </td>
+              </tr>
+            ))}
+            {!loading && !data?.plans.length ? (
+              <tr>
+                <td colSpan={7} style={{ padding: 16, textAlign: 'center', color: '#68716c' }}>
+                  当前时间范围内没有 Dry-run 路由决策。
+                </td>
+              </tr>
+            ) : null}
+          </tbody>
+        </table>
+      </div>
+    </div>
+  );
+}
@@ -7,6 +7,7 @@ import {
  type OrchestratorConfigState,
  type OrchestratorServiceHealth,
 } from '../../api/admin';
+import { OrchestratorExecutionPlanPanel } from './OrchestratorExecutionPlanPanel';
 import { OrchestratorShadowPanel } from './OrchestratorShadowPanel';

 function listToText(values: string[]) {
@@ -170,6 +171,8 @@ export function OrchestratorPage() {

      <OrchestratorShadowPanel />

+      <OrchestratorExecutionPlanPanel />
+
      <div className="card grid">
        <h3 style={{ margin: 0 }}>路由模式</h3>
        <div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(220px, 1fr))', gap: 16 }}>