feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
This commit is contained in:
john
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
+14 -2
View File
@@ -222,6 +222,12 @@ export type OrchestratorRuntime = {
executesLangGraph: boolean;
plansLangGraph: boolean;
shadowsLangGraph: boolean;
shadowObservation: {
requested: boolean;
enabled: boolean;
reason: string | null;
environmentGate: boolean;
};
executionHandoff: {
implemented: boolean;
requested: boolean;
@@ -286,8 +292,10 @@ export type OrchestratorShadowMetrics = {
observations: number;
successes: number;
failures: number;
skipped: number;
successRate: number | null;
failureRate: number | null;
skipRate: number | null;
latencyP50Ms: number | null;
latencyP95Ms: number | null;
nativeSucceeded: number;
@@ -304,7 +312,7 @@ export type OrchestratorShadowRun = {
sessionId: string | null;
nativeStatus: string;
nativeAttempts: number;
shadowStatus: 'succeeded' | 'failed';
shadowStatus: 'succeeded' | 'failed' | 'skipped';
engine: string;
configVersion: number | null;
phase: string | null;
@@ -313,6 +321,7 @@ export type OrchestratorShadowRun = {
executorAdapter: string | null;
latencyMs: number | null;
error: { code: string; message: string } | null;
skipReason: string | null;
observedAt: number;
nativeCompletedAt: number | null;
};
@@ -386,6 +395,7 @@ export type OrchestratorCanaryReadiness = {
hours: number;
minObservations: number;
minSuccessRate: number;
maxSkipRate: number;
maxP95LatencyMs: number;
minLatencyCoverageRate: number;
minNativeSettledRate: number;
@@ -398,7 +408,9 @@ export type OrchestratorCanaryReadiness = {
excludedSynthetic: number;
successes: number;
failures: number;
skipped: number;
successRate: number | null;
skipRate: number | null;
latencyCoverageRate: number | null;
latencyP95Ms: number | null;
nativeSettledRate: number | null;
@@ -521,7 +533,7 @@ export async function fetchOrchestratorRuntime() {
export async function fetchOrchestratorShadowRuns(params: {
hours?: number;
limit?: number;
status?: 'all' | 'succeeded' | 'failed';
status?: 'all' | 'succeeded' | 'failed' | 'skipped';
} = {}) {
const query = new URLSearchParams();
if (params.hours) query.set('hours', String(params.hours));
+19 -1
View File
@@ -25,6 +25,7 @@ function formatTime(value?: number | null) {
const RUNTIME_REASON: Record<string, string> = {
mode_off: '当前为关闭模式,所有任务继续由 Native Agent Run 执行。',
service_url_missing: '尚未配置 Orchestrator 服务地址,LangGraph 不会接管任务。',
service_url_not_allowed: 'Orchestrator 地址不在环境 origin 白名单中,Portal 不会访问该地址。',
kill_switch: '环境级紧急熔断已开启,强制回退 Native。',
execution_gate_disabled: 'Canary / Active 仅生成 Dry-run 候选决策;执行交接闸门仍被硬锁定,Native 是唯一执行者。',
environment_execution_gate_disabled: 'memindadm 已请求执行交接,但环境级执行闸门仍关闭;当前继续 Dry-run。',
@@ -122,11 +123,19 @@ export function OrchestratorPage() {
const runtimeMessage = state.runtime.reason
? (RUNTIME_REASON[state.runtime.reason] ?? state.runtime.reason)
: state.runtime.shadowsLangGraph
: state.runtime.shadowObservation.requested
&& !state.runtime.shadowObservation.enabled
? 'Shadow 配置已保存,但 Portal 环境 wiring 仍关闭;现有 Agent Run 不会查询或调用 Orchestrator。'
: state.runtime.shadowsLangGraph
? 'Shadow 已生效:Native 继续执行,LangGraph 只观察和生成决策。'
: state.runtime.executesLangGraph
? '执行交接已开启。'
: '配置有效。';
const shadowWiringLabel = state.runtime.shadowObservation.enabled
? '已开启'
: state.runtime.shadowObservation.requested
? '环境闸门关闭'
: '未请求';
return (
<div className="grid">
@@ -164,6 +173,15 @@ export function OrchestratorPage() {
{state.runtime.executionHandoff.enabled ? '已开启' : 'Dry-run 锁定'}
</p>
</div>
<div>
<strong>Shadow wiring</strong>
<p style={{
marginBottom: 0,
color: state.runtime.shadowObservation.enabled ? '#2f6f57' : '#8a5a16',
}}>
{shadowWiringLabel}
</p>
</div>
</div>
<p className={state.runtime.reason === 'kill_switch' ? 'alert' : 'warn'} style={{ marginBottom: 0 }}>
{runtimeMessage}
@@ -27,12 +27,14 @@ function shortId(value: string) {
const readinessCheckLabels: Record<string, string> = {
shadow_mode: '当前保持 Shadow',
shadow_wiring_enabled: 'Portal Shadow wiring',
service_healthy: '服务健康',
durable_checkpoint: '持久化 checkpoint',
durable_executor_job_store: '持久化 Executor Job Store',
observe_only: '仅观察不执行',
sample_volume: '有效样本量',
shadow_success_rate: 'Shadow 成功率',
shadow_skip_rate: 'Shadow 跳过率',
latency_coverage: '延迟数据覆盖率',
latency_p95: '延迟 P95',
native_settled_rate: 'Native 终态覆盖率',
@@ -44,7 +46,7 @@ const readinessCheckLabels: Record<string, string> = {
function formatReadinessValue(check: OrchestratorCanaryReadinessCheck) {
const value = check.actual;
if (value == null) return '无数据';
if (['shadow_success_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
if (['shadow_success_rate', 'shadow_skip_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
return formatPercent(Number(value));
}
if (check.id === 'latency_p95') return formatLatency(Number(value));
@@ -56,6 +58,7 @@ function formatReadinessValue(check: OrchestratorCanaryReadinessCheck) {
function formatReadinessTarget(check: OrchestratorCanaryReadinessCheck) {
const target = check.target;
if (target == null) return '';
if (check.id === 'shadow_skip_rate') return `${formatPercent(Number(target))}`;
if (['shadow_success_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
return `${formatPercent(Number(target))}`;
}
@@ -75,7 +78,7 @@ const metricCardStyle = {
export function OrchestratorShadowPanel() {
const [hours, setHours] = useState(24);
const [status, setStatus] = useState<'all' | 'succeeded' | 'failed'>('all');
const [status, setStatus] = useState<'all' | 'succeeded' | 'failed' | 'skipped'>('all');
const [data, setData] = useState<OrchestratorShadowRunList | null>(null);
const [readiness, setReadiness] = useState<OrchestratorCanaryReadiness | null>(null);
const [selected, setSelected] = useState<OrchestratorShadowRunDetail | null>(null);
@@ -141,6 +144,7 @@ export function OrchestratorShadowPanel() {
<option value="all"></option>
<option value="succeeded"></option>
<option value="failed"></option>
<option value="skipped"></option>
</select>
<button type="button" className="btn secondary" onClick={() => void load()} disabled={loading}>
{loading ? '刷新中…' : '刷新'}
@@ -206,6 +210,12 @@ export function OrchestratorShadowPanel() {
<small></small>
<div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.failures ?? '—'}</div>
</div>
<div style={metricCardStyle}>
<small> / </small>
<div style={{ fontSize: 20, marginTop: 7 }}>
{metrics ? `${metrics.skipped} / ${formatPercent(metrics.skipRate)}` : '—'}
</div>
</div>
<div style={metricCardStyle}>
<small> P50</small>
<div style={{ fontSize: 24, marginTop: 4 }}>{formatLatency(metrics?.latencyP50Ms ?? null)}</div>
@@ -254,7 +264,14 @@ export function OrchestratorShadowPanel() {
{shortId(run.runId)}
</button>
</td>
<td style={{ padding: 8, color: run.shadowStatus === 'succeeded' ? '#2f6f57' : '#a23a32' }}>
<td style={{
padding: 8,
color: run.shadowStatus === 'succeeded'
? '#2f6f57'
: run.shadowStatus === 'skipped'
? '#8a5a16'
: '#a23a32',
}}>
{run.shadowStatus}
</td>
<td style={{ padding: 8 }}>{run.nativeStatus}</td>
@@ -267,7 +284,9 @@ export function OrchestratorShadowPanel() {
</small>
</td>
<td style={{ padding: 8, maxWidth: 260 }}>
{run.error ? `${run.error.code}: ${run.error.message}` : '—'}
{run.error
? `${run.error.code}: ${run.error.message}`
: run.skipReason ?? '—'}
</td>
</tr>
))}