feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.

services/orchestrator/observability.mjs

@@ -4,6 +4,7 @@ import { WORKFLOW_ENGINE } from './contracts.mjs';
 const SHADOW_EVENT_TYPES = Object.freeze([
   'workflow_shadow_completed',
   'workflow_shadow_failed',
+  'workflow_shadow_skipped',
 ]);
 const EXECUTION_PLAN_EVENT_TYPE = 'workflow_execution_planned';
 const MAX_METRIC_EVENTS = 5000;
@@ -11,6 +12,7 @@ const CANARY_READINESS_THRESHOLDS = Object.freeze({
   hours: 24 * 7,
   minObservations: 20,
   minSuccessRate: 0.95,
+  maxSkipRate: 0,
   maxP95LatencyMs: 2000,
   minLatencyCoverageRate: 0.95,
   minNativeSettledRate: 0.95,
@@ -56,6 +58,7 @@ function normalizeExecutorJobId(value) {
 function projectShadowEventRow(row) {
   const data = parseJsonColumn(row.data_json, {}) ?? {};
   const succeeded = row.event_type === 'workflow_shadow_completed';
+  const skipped = row.event_type === 'workflow_shadow_skipped';
   const latency = Number(data.latencyMs);
   const taskType = data.taskType ?? null;
   return {
@@ -66,7 +69,7 @@ function projectShadowEventRow(row) {
     sessionId: row.agent_session_id ?? null,
     nativeStatus: row.native_status,
     nativeAttempts: Number(row.native_attempts ?? 0),
-    shadowStatus: succeeded ? 'succeeded' : 'failed',
+    shadowStatus: succeeded ? 'succeeded' : skipped ? 'skipped' : 'failed',
     engine: data.engine ?? 'langgraph',
     configVersion: data.configVersion == null ? null : Number(data.configVersion),
     phase: data.phase ?? null,
@@ -74,10 +77,11 @@ function projectShadowEventRow(row) {
     synthetic: data.synthetic === true || SYNTHETIC_TASK_TYPES.has(String(taskType ?? '')),
     executorAdapter: data.executorAdapter ?? null,
     latencyMs: Number.isFinite(latency) && latency >= 0 ? latency : null,
-    error: succeeded ? null : {
+    error: succeeded || skipped ? null : {
       code: data.code ?? 'WORKFLOW_SHADOW_FAILED',
       message: data.message ?? 'Shadow observation failed',
     },
+    skipReason: skipped ? data.reason ?? 'shadow_queue_full' : null,
     observedAt: Number(row.event_created_at ?? 0),
     nativeCompletedAt: row.native_completed_at == null
       ? null
@@ -146,6 +150,8 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
   const thresholds = CANARY_READINESS_THRESHOLDS;
   const eligibleRuns = loaded.runs.filter((run) => !run.synthetic);
   const successes = eligibleRuns.filter((run) => run.shadowStatus === 'succeeded').length;
+  const failures = eligibleRuns.filter((run) => run.shadowStatus === 'failed').length;
+  const skipped = eligibleRuns.filter((run) => run.shadowStatus === 'skipped').length;
   const latencies = eligibleRuns
     .map((run) => run.latencyMs)
     .filter((value) => Number.isFinite(value));
@@ -159,6 +165,7 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
     ? null
     : Math.max(0, (now - lastObservedAt) / (60 * 60 * 1000));
   const successRate = ratio(successes, eligibleRuns.length);
+  const skipRate = ratio(skipped, eligibleRuns.length);
   const latencyCoverageRate = ratio(latencies.length, eligibleRuns.length);
   const nativeSettledRate = ratio(nativeSettled, eligibleRuns.length);
   const latencyP95Ms = percentile(latencies, 0.95);
@@ -173,6 +180,12 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
       actual: runtimeState?.config?.mode ?? null,
       target: 'shadow',
     },
+    {
+      id: 'shadow_wiring_enabled',
+      passed: runtimeState?.runtime?.shadowObservation?.enabled === true,
+      actual: runtimeState?.runtime?.shadowObservation?.enabled ?? null,
+      target: true,
+    },
     {
       id: 'service_healthy',
       passed: serviceHealth?.ok === true,
@@ -209,6 +222,12 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
       actual: successRate,
       target: thresholds.minSuccessRate,
     },
+    {
+      id: 'shadow_skip_rate',
+      passed: skipRate != null && skipRate <= thresholds.maxSkipRate,
+      actual: skipRate,
+      target: thresholds.maxSkipRate,
+    },
     {
       id: 'latency_coverage',
       passed: latencyCoverageRate != null
@@ -271,8 +290,10 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
       eligibleObservations: eligibleRuns.length,
       excludedSynthetic: loaded.runs.length - eligibleRuns.length,
       successes,
-      failures: eligibleRuns.length - successes,
+      failures,
+      skipped,
       successRate,
+      skipRate,
       latencyCoverageRate,
       latencyP95Ms,
       nativeSettledRate,
@@ -301,7 +322,8 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
 
 function summarizeRuns(runs, { capped = false } = {}) {
   const successes = runs.filter((run) => run.shadowStatus === 'succeeded').length;
-  const failures = runs.length - successes;
+  const failures = runs.filter((run) => run.shadowStatus === 'failed').length;
+  const skipped = runs.filter((run) => run.shadowStatus === 'skipped').length;
   const latencies = runs
     .map((run) => run.latencyMs)
     .filter((value) => Number.isFinite(value));
@@ -309,8 +331,10 @@ function summarizeRuns(runs, { capped = false } = {}) {
     observations: runs.length,
     successes,
     failures,
+    skipped,
     successRate: runs.length ? successes / runs.length : null,
     failureRate: runs.length ? failures / runs.length : null,
+    skipRate: runs.length ? skipped / runs.length : null,
     latencyP50Ms: percentile(latencies, 0.5),
     latencyP95Ms: percentile(latencies, 0.95),
     nativeSucceeded: runs.filter((run) => run.nativeStatus === 'succeeded').length,
@@ -391,7 +415,7 @@ export function createOrchestratorObservabilityService({
          r.completed_at AS native_completed_at
        FROM h5_agent_run_events e
        INNER JOIN h5_agent_runs r ON r.id = e.run_id
-       WHERE e.event_type IN (?, ?)
+       WHERE e.event_type IN (${SHADOW_EVENT_TYPES.map(() => '?').join(', ')})
          AND e.created_at >= ?
        ORDER BY e.created_at DESC, e.id DESC
        LIMIT ?`,
@@ -452,7 +476,9 @@ export function createOrchestratorObservabilityService({
     } = {}) {
       const loaded = await loadShadowEvents({ hours });
       const normalizedLimit = clampInteger(limit, 50, 1, 200);
-      const normalizedStatus = ['succeeded', 'failed'].includes(status) ? status : 'all';
+      const normalizedStatus = ['succeeded', 'failed', 'skipped'].includes(status)
+        ? status
+        : 'all';
       const filtered = normalizedStatus === 'all'
         ? loaded.runs
         : loaded.runs.filter((run) => run.shadowStatus === normalizedStatus);
@@ -510,7 +536,8 @@ export function createOrchestratorObservabilityService({
       const [eventRows] = await pool.query(
         `SELECT id AS event_id, event_type, data_json, created_at
            FROM h5_agent_run_events
-          WHERE run_id = ? AND event_type IN (?, ?)
+          WHERE run_id = ?
+            AND event_type IN (${SHADOW_EVENT_TYPES.map(() => '?').join(', ')})
           ORDER BY created_at ASC, id ASC`,
         [normalizedRunId, ...SHADOW_EVENT_TYPES],
       );