feat: add orchestrator canary readiness gate

2026-07-24 22:09:19 +08:00
parent ff9c68e57f
commit 85e888b340
8 changed files with 534 additions and 6 deletions
@@ -1,6 +1,9 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
-import { createOrchestratorObservabilityService } from './observability.mjs';
+import {
+  createOrchestratorObservabilityService,
+  orchestratorObservabilityInternals,
+} from './observability.mjs';

 function response(body, status = 200) {
  return new Response(JSON.stringify(body), {
@@ -148,3 +151,145 @@ test('observability rejects unsafe run ids before querying', async () => {
  assert.equal(await service.getShadowRun('../unsafe'), null);
  assert.equal(queries, 0);
 });
+
+test('observability counts only the newest terminal event for each run', () => {
+  const base = {
+    run_id: 'same-run',
+    request_id: 'same-request',
+    user_id: 'same-user',
+    native_status: 'succeeded',
+    native_attempts: 1,
+  };
+  const runs = orchestratorObservabilityInternals.projectUniqueShadowRuns([
+    {
+      ...base,
+      event_id: 'newer-success',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 80 },
+      event_created_at: 2000,
+    },
+    {
+      ...base,
+      event_id: 'older-failure',
+      event_type: 'workflow_shadow_failed',
+      data_json: { latencyMs: 100, code: 'TIMEOUT' },
+      event_created_at: 1000,
+    },
+  ]);
+  assert.equal(runs.length, 1);
+  assert.equal(runs[0].eventId, 'newer-success');
+  assert.equal(runs[0].shadowStatus, 'succeeded');
+});
+
+test('canary readiness excludes smoke runs and reports explicit blockers', async () => {
+  const now = 10_000_000;
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        return [[{
+          event_id: 'smoke-event',
+          run_id: 'smoke-run',
+          event_type: 'workflow_shadow_completed',
+          data_json: {
+            latencyMs: 50,
+            taskType: 'orchestrator_shadow_smoke',
+          },
+          event_created_at: now - 1000,
+          request_id: 'smoke-request',
+          user_id: 'smoke-user',
+          agent_session_id: null,
+          native_status: 'succeeded',
+          native_attempts: 0,
+          native_completed_at: now,
+        }]];
+      },
+    },
+    configService: {
+      async getRuntimeState(options) {
+        assert.deepEqual(options, { probe: true });
+        return {
+          config: { mode: 'shadow' },
+          serviceHealth: {
+            ok: true,
+            status: 'healthy',
+            latencyMs: 5,
+            details: {
+              checkpoint: { kind: 'postgres', durable: true },
+              execution: 'observe-only',
+            },
+          },
+        };
+      },
+    },
+    nowMs: () => now,
+  });
+
+  const readiness = await service.getCanaryReadiness();
+  assert.equal(readiness.ready, false);
+  assert.equal(readiness.recommendation, 'keep_shadow');
+  assert.equal(readiness.samples.totalObservations, 1);
+  assert.equal(readiness.samples.eligibleObservations, 0);
+  assert.equal(readiness.samples.excludedSynthetic, 1);
+  assert.ok(readiness.blockers.includes('sample_volume'));
+  assert.ok(readiness.blockers.includes('session_coverage'));
+  assert.ok(!readiness.blockers.includes('service_healthy'));
+});
+
+test('canary readiness passes only when operational and sample gates all pass', async () => {
+  const now = 20_000_000;
+  const rows = Array.from({ length: 20 }, (_, index) => ({
+    event_id: `event-${index}`,
+    run_id: `run-${index}`,
+    event_type: index === 19
+      ? 'workflow_shadow_failed'
+      : 'workflow_shadow_completed',
+    data_json: {
+      latencyMs: 100 + index,
+      taskType: 'code_task',
+      ...(index === 19 ? { code: 'TRANSIENT', message: 'retry later' } : {}),
+    },
+    event_created_at: now - index * 1000,
+    request_id: `request-${index}`,
+    user_id: `user-${index % 3}`,
+    agent_session_id: `session-${index % 5}`,
+    native_status: index % 2 ? 'succeeded' : 'failed',
+    native_attempts: 1,
+    native_completed_at: now,
+  }));
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        return [rows];
+      },
+    },
+    configService: {
+      async getRuntimeState() {
+        return {
+          config: { mode: 'shadow' },
+          serviceHealth: {
+            ok: true,
+            status: 'healthy',
+            latencyMs: 4,
+            details: {
+              checkpoint: { kind: 'postgres', durable: true },
+              execution: 'observe-only',
+            },
+          },
+        };
+      },
+    },
+    nowMs: () => now,
+  });
+
+  const readiness = await service.getCanaryReadiness();
+  assert.equal(readiness.ready, true);
+  assert.equal(readiness.recommendation, 'manual_canary_review');
+  assert.equal(readiness.window.hours, 168);
+  assert.equal(readiness.samples.eligibleObservations, 20);
+  assert.equal(readiness.samples.successRate, 0.95);
+  assert.equal(readiness.samples.latencyCoverageRate, 1);
+  assert.equal(readiness.samples.nativeSettledRate, 1);
+  assert.equal(readiness.samples.distinctSessions, 5);
+  assert.equal(readiness.blockers.length, 0);
+  assert.deepEqual(readiness.failureCodes, [{ code: 'TRANSIENT', count: 1 }]);
+});