feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.

services/orchestrator/observability.test.mjs

@@ -50,6 +50,18 @@ test('observability aggregates shadow outcomes and latency percentiles', async (
       native_attempts: 2,
       native_completed_at: 1500,
     },
+    {
+      event_id: 'event-skipped',
+      run_id: 'run-skipped',
+      event_type: 'workflow_shadow_skipped',
+      data_json: { reason: 'shadow_queue_full' },
+      event_created_at: 900,
+      request_id: 'request-skipped',
+      user_id: 'user-skipped',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 1200,
+    },
   ];
   const service = createOrchestratorObservabilityService({
     pool: {
@@ -61,16 +73,20 @@ test('observability aggregates shadow outcomes and latency percentiles', async (
     configService: { getRuntimeState() {} },
   });
 
-  const result = await service.listShadowRuns({ hours: 12, limit: 2 });
+  const result = await service.listShadowRuns({ hours: 12, limit: 4 });
   assert.equal(result.window.hours, 12);
-  assert.equal(result.metrics.observations, 3);
+  assert.equal(result.metrics.observations, 4);
   assert.equal(result.metrics.successes, 2);
   assert.equal(result.metrics.failures, 1);
+  assert.equal(result.metrics.skipped, 1);
+  assert.equal(result.metrics.skipRate, 0.25);
   assert.equal(result.metrics.latencyP50Ms, 200);
   assert.equal(result.metrics.latencyP95Ms, 300);
-  assert.equal(result.metrics.nativeSucceeded, 2);
-  assert.equal(result.runs.length, 2);
+  assert.equal(result.metrics.nativeSucceeded, 3);
+  assert.equal(result.runs.length, 4);
   assert.equal(result.runs[1].error.code, 'TIMEOUT');
+  assert.equal(result.runs[3].shadowStatus, 'skipped');
+  assert.equal(result.runs[3].skipReason, 'shadow_queue_full');
 });
 
 test('observability aggregates complete dry-run routing decisions and Native outcomes', async () => {
@@ -339,6 +355,9 @@ test('canary readiness excludes smoke runs and reports explicit blockers', async
         assert.deepEqual(options, { probe: true });
         return {
           config: { mode: 'shadow' },
+          runtime: {
+            shadowObservation: { enabled: true },
+          },
           serviceHealth: {
             ok: true,
             status: 'healthy',
@@ -370,6 +389,7 @@ test('canary readiness excludes smoke runs and reports explicit blockers', async
 
 test('canary readiness passes only when operational and sample gates all pass', async () => {
   const now = 20_000_000;
+  let shadowObservationEnabled = true;
   const rows = Array.from({ length: 20 }, (_, index) => ({
     event_id: `event-${index}`,
     run_id: `run-${index}`,
@@ -399,6 +419,9 @@ test('canary readiness passes only when operational and sample gates all pass',
       async getRuntimeState() {
         return {
           config: { mode: 'shadow' },
+          runtime: {
+            shadowObservation: { enabled: shadowObservationEnabled },
+          },
           serviceHealth: {
             ok: true,
             status: 'healthy',
@@ -423,10 +446,67 @@ test('canary readiness passes only when operational and sample gates all pass',
   assert.equal(readiness.window.hours, 168);
   assert.equal(readiness.samples.eligibleObservations, 20);
   assert.equal(readiness.samples.successRate, 0.95);
+  assert.equal(readiness.samples.skipped, 0);
+  assert.equal(readiness.samples.skipRate, 0);
   assert.equal(readiness.samples.latencyCoverageRate, 1);
   assert.equal(readiness.samples.nativeSettledRate, 1);
   assert.equal(readiness.samples.distinctSessions, 5);
   assert.equal(readiness.service.executorJobStoreDurable, true);
   assert.equal(readiness.blockers.length, 0);
   assert.deepEqual(readiness.failureCodes, [{ code: 'TRANSIENT', count: 1 }]);
+
+  shadowObservationEnabled = false;
+  const blockedReadiness = await service.getCanaryReadiness();
+  assert.equal(blockedReadiness.ready, false);
+  assert.ok(blockedReadiness.blockers.includes('shadow_wiring_enabled'));
+});
+
+test('canary readiness blocks when any Shadow observation was skipped', async () => {
+  const now = 30_000_000;
+  const rows = Array.from({ length: 20 }, (_, index) => ({
+    event_id: `event-skip-${index}`,
+    run_id: `run-skip-${index}`,
+    event_type: index === 0
+      ? 'workflow_shadow_skipped'
+      : 'workflow_shadow_completed',
+    data_json: index === 0
+      ? { reason: 'shadow_queue_full' }
+      : { latencyMs: 100, taskType: 'code_task' },
+    event_created_at: now - index * 1000,
+    request_id: `request-skip-${index}`,
+    user_id: `user-${index % 3}`,
+    agent_session_id: `session-${index % 5}`,
+    native_status: 'succeeded',
+    native_attempts: 1,
+    native_completed_at: now,
+  }));
+  const service = createOrchestratorObservabilityService({
+    pool: { async query() { return [rows]; } },
+    configService: {
+      async getRuntimeState() {
+        return {
+          config: { mode: 'shadow' },
+          runtime: {
+            shadowObservation: { enabled: true },
+          },
+          serviceHealth: {
+            ok: true,
+            status: 'healthy',
+            details: {
+              checkpoint: { kind: 'postgres', durable: true },
+              executorGateway: { store: { kind: 'postgres', durable: true } },
+              execution: 'observe-only',
+            },
+          },
+        };
+      },
+    },
+    nowMs: () => now,
+  });
+
+  const readiness = await service.getCanaryReadiness();
+  assert.equal(readiness.ready, false);
+  assert.equal(readiness.samples.skipped, 1);
+  assert.equal(readiness.samples.skipRate, 0.05);
+  assert.ok(readiness.blockers.includes('shadow_skip_rate'));
 });