feat(orchestrator): harden zero-impact shadow rollout
Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
This commit is contained in:
+14
-2
@@ -222,6 +222,12 @@ export type OrchestratorRuntime = {
|
||||
executesLangGraph: boolean;
|
||||
plansLangGraph: boolean;
|
||||
shadowsLangGraph: boolean;
|
||||
shadowObservation: {
|
||||
requested: boolean;
|
||||
enabled: boolean;
|
||||
reason: string | null;
|
||||
environmentGate: boolean;
|
||||
};
|
||||
executionHandoff: {
|
||||
implemented: boolean;
|
||||
requested: boolean;
|
||||
@@ -286,8 +292,10 @@ export type OrchestratorShadowMetrics = {
|
||||
observations: number;
|
||||
successes: number;
|
||||
failures: number;
|
||||
skipped: number;
|
||||
successRate: number | null;
|
||||
failureRate: number | null;
|
||||
skipRate: number | null;
|
||||
latencyP50Ms: number | null;
|
||||
latencyP95Ms: number | null;
|
||||
nativeSucceeded: number;
|
||||
@@ -304,7 +312,7 @@ export type OrchestratorShadowRun = {
|
||||
sessionId: string | null;
|
||||
nativeStatus: string;
|
||||
nativeAttempts: number;
|
||||
shadowStatus: 'succeeded' | 'failed';
|
||||
shadowStatus: 'succeeded' | 'failed' | 'skipped';
|
||||
engine: string;
|
||||
configVersion: number | null;
|
||||
phase: string | null;
|
||||
@@ -313,6 +321,7 @@ export type OrchestratorShadowRun = {
|
||||
executorAdapter: string | null;
|
||||
latencyMs: number | null;
|
||||
error: { code: string; message: string } | null;
|
||||
skipReason: string | null;
|
||||
observedAt: number;
|
||||
nativeCompletedAt: number | null;
|
||||
};
|
||||
@@ -386,6 +395,7 @@ export type OrchestratorCanaryReadiness = {
|
||||
hours: number;
|
||||
minObservations: number;
|
||||
minSuccessRate: number;
|
||||
maxSkipRate: number;
|
||||
maxP95LatencyMs: number;
|
||||
minLatencyCoverageRate: number;
|
||||
minNativeSettledRate: number;
|
||||
@@ -398,7 +408,9 @@ export type OrchestratorCanaryReadiness = {
|
||||
excludedSynthetic: number;
|
||||
successes: number;
|
||||
failures: number;
|
||||
skipped: number;
|
||||
successRate: number | null;
|
||||
skipRate: number | null;
|
||||
latencyCoverageRate: number | null;
|
||||
latencyP95Ms: number | null;
|
||||
nativeSettledRate: number | null;
|
||||
@@ -521,7 +533,7 @@ export async function fetchOrchestratorRuntime() {
|
||||
export async function fetchOrchestratorShadowRuns(params: {
|
||||
hours?: number;
|
||||
limit?: number;
|
||||
status?: 'all' | 'succeeded' | 'failed';
|
||||
status?: 'all' | 'succeeded' | 'failed' | 'skipped';
|
||||
} = {}) {
|
||||
const query = new URLSearchParams();
|
||||
if (params.hours) query.set('hours', String(params.hours));
|
||||
|
||||
@@ -25,6 +25,7 @@ function formatTime(value?: number | null) {
|
||||
const RUNTIME_REASON: Record<string, string> = {
|
||||
mode_off: '当前为关闭模式,所有任务继续由 Native Agent Run 执行。',
|
||||
service_url_missing: '尚未配置 Orchestrator 服务地址,LangGraph 不会接管任务。',
|
||||
service_url_not_allowed: 'Orchestrator 地址不在环境 origin 白名单中,Portal 不会访问该地址。',
|
||||
kill_switch: '环境级紧急熔断已开启,强制回退 Native。',
|
||||
execution_gate_disabled: 'Canary / Active 仅生成 Dry-run 候选决策;执行交接闸门仍被硬锁定,Native 是唯一执行者。',
|
||||
environment_execution_gate_disabled: 'memindadm 已请求执行交接,但环境级执行闸门仍关闭;当前继续 Dry-run。',
|
||||
@@ -122,11 +123,19 @@ export function OrchestratorPage() {
|
||||
|
||||
const runtimeMessage = state.runtime.reason
|
||||
? (RUNTIME_REASON[state.runtime.reason] ?? state.runtime.reason)
|
||||
: state.runtime.shadowsLangGraph
|
||||
: state.runtime.shadowObservation.requested
|
||||
&& !state.runtime.shadowObservation.enabled
|
||||
? 'Shadow 配置已保存,但 Portal 环境 wiring 仍关闭;现有 Agent Run 不会查询或调用 Orchestrator。'
|
||||
: state.runtime.shadowsLangGraph
|
||||
? 'Shadow 已生效:Native 继续执行,LangGraph 只观察和生成决策。'
|
||||
: state.runtime.executesLangGraph
|
||||
? '执行交接已开启。'
|
||||
: '配置有效。';
|
||||
const shadowWiringLabel = state.runtime.shadowObservation.enabled
|
||||
? '已开启'
|
||||
: state.runtime.shadowObservation.requested
|
||||
? '环境闸门关闭'
|
||||
: '未请求';
|
||||
|
||||
return (
|
||||
<div className="grid">
|
||||
@@ -164,6 +173,15 @@ export function OrchestratorPage() {
|
||||
{state.runtime.executionHandoff.enabled ? '已开启' : 'Dry-run 锁定'}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<strong>Shadow wiring</strong>
|
||||
<p style={{
|
||||
marginBottom: 0,
|
||||
color: state.runtime.shadowObservation.enabled ? '#2f6f57' : '#8a5a16',
|
||||
}}>
|
||||
{shadowWiringLabel}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<p className={state.runtime.reason === 'kill_switch' ? 'alert' : 'warn'} style={{ marginBottom: 0 }}>
|
||||
{runtimeMessage}
|
||||
|
||||
@@ -27,12 +27,14 @@ function shortId(value: string) {
|
||||
|
||||
const readinessCheckLabels: Record<string, string> = {
|
||||
shadow_mode: '当前保持 Shadow',
|
||||
shadow_wiring_enabled: 'Portal Shadow wiring',
|
||||
service_healthy: '服务健康',
|
||||
durable_checkpoint: '持久化 checkpoint',
|
||||
durable_executor_job_store: '持久化 Executor Job Store',
|
||||
observe_only: '仅观察不执行',
|
||||
sample_volume: '有效样本量',
|
||||
shadow_success_rate: 'Shadow 成功率',
|
||||
shadow_skip_rate: 'Shadow 跳过率',
|
||||
latency_coverage: '延迟数据覆盖率',
|
||||
latency_p95: '延迟 P95',
|
||||
native_settled_rate: 'Native 终态覆盖率',
|
||||
@@ -44,7 +46,7 @@ const readinessCheckLabels: Record<string, string> = {
|
||||
function formatReadinessValue(check: OrchestratorCanaryReadinessCheck) {
|
||||
const value = check.actual;
|
||||
if (value == null) return '无数据';
|
||||
if (['shadow_success_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
|
||||
if (['shadow_success_rate', 'shadow_skip_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
|
||||
return formatPercent(Number(value));
|
||||
}
|
||||
if (check.id === 'latency_p95') return formatLatency(Number(value));
|
||||
@@ -56,6 +58,7 @@ function formatReadinessValue(check: OrchestratorCanaryReadinessCheck) {
|
||||
function formatReadinessTarget(check: OrchestratorCanaryReadinessCheck) {
|
||||
const target = check.target;
|
||||
if (target == null) return '';
|
||||
if (check.id === 'shadow_skip_rate') return `≤ ${formatPercent(Number(target))}`;
|
||||
if (['shadow_success_rate', 'latency_coverage', 'native_settled_rate'].includes(check.id)) {
|
||||
return `≥ ${formatPercent(Number(target))}`;
|
||||
}
|
||||
@@ -75,7 +78,7 @@ const metricCardStyle = {
|
||||
|
||||
export function OrchestratorShadowPanel() {
|
||||
const [hours, setHours] = useState(24);
|
||||
const [status, setStatus] = useState<'all' | 'succeeded' | 'failed'>('all');
|
||||
const [status, setStatus] = useState<'all' | 'succeeded' | 'failed' | 'skipped'>('all');
|
||||
const [data, setData] = useState<OrchestratorShadowRunList | null>(null);
|
||||
const [readiness, setReadiness] = useState<OrchestratorCanaryReadiness | null>(null);
|
||||
const [selected, setSelected] = useState<OrchestratorShadowRunDetail | null>(null);
|
||||
@@ -141,6 +144,7 @@ export function OrchestratorShadowPanel() {
|
||||
<option value="all">全部结果</option>
|
||||
<option value="succeeded">仅成功</option>
|
||||
<option value="failed">仅失败</option>
|
||||
<option value="skipped">仅跳过</option>
|
||||
</select>
|
||||
<button type="button" className="btn secondary" onClick={() => void load()} disabled={loading}>
|
||||
{loading ? '刷新中…' : '刷新'}
|
||||
@@ -206,6 +210,12 @@ export function OrchestratorShadowPanel() {
|
||||
<small>失败数</small>
|
||||
<div style={{ fontSize: 24, marginTop: 4 }}>{metrics?.failures ?? '—'}</div>
|
||||
</div>
|
||||
<div style={metricCardStyle}>
|
||||
<small>跳过数 / 跳过率</small>
|
||||
<div style={{ fontSize: 20, marginTop: 7 }}>
|
||||
{metrics ? `${metrics.skipped} / ${formatPercent(metrics.skipRate)}` : '—'}
|
||||
</div>
|
||||
</div>
|
||||
<div style={metricCardStyle}>
|
||||
<small>延迟 P50</small>
|
||||
<div style={{ fontSize: 24, marginTop: 4 }}>{formatLatency(metrics?.latencyP50Ms ?? null)}</div>
|
||||
@@ -254,7 +264,14 @@ export function OrchestratorShadowPanel() {
|
||||
{shortId(run.runId)}
|
||||
</button>
|
||||
</td>
|
||||
<td style={{ padding: 8, color: run.shadowStatus === 'succeeded' ? '#2f6f57' : '#a23a32' }}>
|
||||
<td style={{
|
||||
padding: 8,
|
||||
color: run.shadowStatus === 'succeeded'
|
||||
? '#2f6f57'
|
||||
: run.shadowStatus === 'skipped'
|
||||
? '#8a5a16'
|
||||
: '#a23a32',
|
||||
}}>
|
||||
{run.shadowStatus}
|
||||
</td>
|
||||
<td style={{ padding: 8 }}>{run.nativeStatus}</td>
|
||||
@@ -267,7 +284,9 @@ export function OrchestratorShadowPanel() {
|
||||
</small>
|
||||
</td>
|
||||
<td style={{ padding: 8, maxWidth: 260 }}>
|
||||
{run.error ? `${run.error.code}: ${run.error.message}` : '—'}
|
||||
{run.error
|
||||
? `${run.error.code}: ${run.error.message}`
|
||||
: run.skipReason ?? '—'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
|
||||
Reference in New Issue
Block a user