feat: add orchestrator canary readiness gate

services/orchestrator/README.md

@@ -60,6 +60,18 @@ Phase 2.5 adds a Memind-owned Shadow observability projection:
 The projection reads `h5_agent_run_events` from the Memind control plane.
 LangGraph still has no access to Memind business tables.
 
+Phase 2.6 adds a read-only Canary readiness gate. It evaluates a fixed seven-day
+window and never changes the runtime mode. The first gate requires:
+
+- at least 20 non-smoke observations across at least five real sessions;
+- at least 95% Shadow success, latency coverage, and Native terminal coverage;
+- Shadow P95 latency no greater than two seconds;
+- a healthy observe-only service backed by a durable checkpoint;
+- a fresh, uncapped metric window while the control plane remains in Shadow.
+
+Passing the gate means only `manual_canary_review`; it does not authorize or
+activate Canary routing.
+
 The default memindadm mode remains `off`. Canary and Active routing are not
 wired to the LangGraph executor in Phase 2. Native Agent Run remains the sole
 executor, including in Shadow mode.
@@ -101,6 +113,7 @@ memindadm exposes the projection through:
 ```text
 GET /admin-api/orchestrator/shadow-runs
 GET /admin-api/orchestrator/shadow-runs/:runId
+GET /admin-api/orchestrator/canary-readiness
 ```
 
 ## Colima deployment

services/orchestrator/observability.mjs

@@ -5,6 +5,18 @@ const SHADOW_EVENT_TYPES = Object.freeze([
   'workflow_shadow_failed',
 ]);
 const MAX_METRIC_EVENTS = 5000;
+const CANARY_READINESS_THRESHOLDS = Object.freeze({
+  hours: 24 * 7,
+  minObservations: 20,
+  minSuccessRate: 0.95,
+  maxP95LatencyMs: 2000,
+  minLatencyCoverageRate: 0.95,
+  minNativeSettledRate: 0.95,
+  minDistinctSessions: 5,
+  maxHoursSinceLastObservation: 24,
+});
+const NATIVE_TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
+const SYNTHETIC_TASK_TYPES = new Set(['orchestrator_shadow_smoke']);
 
 function clampInteger(value, fallback, min, max) {
   const parsed = Number(value);
@@ -38,6 +50,7 @@ function projectShadowEventRow(row) {
   const data = parseJsonColumn(row.data_json, {}) ?? {};
   const succeeded = row.event_type === 'workflow_shadow_completed';
   const latency = Number(data.latencyMs);
+  const taskType = data.taskType ?? null;
   return {
     eventId: row.event_id,
     runId: row.run_id,
@@ -50,7 +63,8 @@ function projectShadowEventRow(row) {
     engine: data.engine ?? 'langgraph',
     configVersion: data.configVersion == null ? null : Number(data.configVersion),
     phase: data.phase ?? null,
-    taskType: data.taskType ?? null,
+    taskType,
+    synthetic: data.synthetic === true || SYNTHETIC_TASK_TYPES.has(String(taskType ?? '')),
     executorAdapter: data.executorAdapter ?? null,
     latencyMs: Number.isFinite(latency) && latency >= 0 ? latency : null,
     error: succeeded ? null : {
@@ -64,6 +78,170 @@ function projectShadowEventRow(row) {
   };
 }
 
+function projectUniqueShadowRuns(rows) {
+  const seen = new Set();
+  const runs = [];
+  for (const row of rows) {
+    const run = projectShadowEventRow(row);
+    if (seen.has(run.runId)) continue;
+    seen.add(run.runId);
+    runs.push(run);
+  }
+  return runs;
+}
+
+function ratio(numerator, denominator) {
+  return denominator > 0 ? numerator / denominator : null;
+}
+
+function buildCanaryReadiness(loaded, runtimeState, now) {
+  const thresholds = CANARY_READINESS_THRESHOLDS;
+  const eligibleRuns = loaded.runs.filter((run) => !run.synthetic);
+  const successes = eligibleRuns.filter((run) => run.shadowStatus === 'succeeded').length;
+  const latencies = eligibleRuns
+    .map((run) => run.latencyMs)
+    .filter((value) => Number.isFinite(value));
+  const nativeSettled = eligibleRuns.filter((run) =>
+    NATIVE_TERMINAL_STATUSES.has(run.nativeStatus)).length;
+  const distinctSessions = new Set(
+    eligibleRuns.map((run) => run.sessionId).filter(Boolean),
+  ).size;
+  const lastObservedAt = eligibleRuns[0]?.observedAt ?? null;
+  const hoursSinceLastObservation = lastObservedAt == null
+    ? null
+    : Math.max(0, (now - lastObservedAt) / (60 * 60 * 1000));
+  const successRate = ratio(successes, eligibleRuns.length);
+  const latencyCoverageRate = ratio(latencies.length, eligibleRuns.length);
+  const nativeSettledRate = ratio(nativeSettled, eligibleRuns.length);
+  const latencyP95Ms = percentile(latencies, 0.95);
+  const serviceHealth = runtimeState?.serviceHealth ?? null;
+  const checkpoint = serviceHealth?.details?.checkpoint ?? null;
+
+  const checks = [
+    {
+      id: 'shadow_mode',
+      passed: runtimeState?.config?.mode === 'shadow',
+      actual: runtimeState?.config?.mode ?? null,
+      target: 'shadow',
+    },
+    {
+      id: 'service_healthy',
+      passed: serviceHealth?.ok === true,
+      actual: serviceHealth?.status ?? null,
+      target: 'healthy',
+    },
+    {
+      id: 'durable_checkpoint',
+      passed: checkpoint?.durable === true,
+      actual: checkpoint?.durable ?? null,
+      target: true,
+    },
+    {
+      id: 'observe_only',
+      passed: serviceHealth?.details?.execution === 'observe-only',
+      actual: serviceHealth?.details?.execution ?? null,
+      target: 'observe-only',
+    },
+    {
+      id: 'sample_volume',
+      passed: eligibleRuns.length >= thresholds.minObservations,
+      actual: eligibleRuns.length,
+      target: thresholds.minObservations,
+    },
+    {
+      id: 'shadow_success_rate',
+      passed: successRate != null && successRate >= thresholds.minSuccessRate,
+      actual: successRate,
+      target: thresholds.minSuccessRate,
+    },
+    {
+      id: 'latency_coverage',
+      passed: latencyCoverageRate != null
+        && latencyCoverageRate >= thresholds.minLatencyCoverageRate,
+      actual: latencyCoverageRate,
+      target: thresholds.minLatencyCoverageRate,
+    },
+    {
+      id: 'latency_p95',
+      passed: latencyP95Ms != null && latencyP95Ms <= thresholds.maxP95LatencyMs,
+      actual: latencyP95Ms,
+      target: thresholds.maxP95LatencyMs,
+    },
+    {
+      id: 'native_settled_rate',
+      passed: nativeSettledRate != null
+        && nativeSettledRate >= thresholds.minNativeSettledRate,
+      actual: nativeSettledRate,
+      target: thresholds.minNativeSettledRate,
+    },
+    {
+      id: 'session_coverage',
+      passed: distinctSessions >= thresholds.minDistinctSessions,
+      actual: distinctSessions,
+      target: thresholds.minDistinctSessions,
+    },
+    {
+      id: 'sample_freshness',
+      passed: hoursSinceLastObservation != null
+        && hoursSinceLastObservation <= thresholds.maxHoursSinceLastObservation,
+      actual: hoursSinceLastObservation,
+      target: thresholds.maxHoursSinceLastObservation,
+    },
+    {
+      id: 'complete_window',
+      passed: !loaded.capped,
+      actual: loaded.capped ? 'sampled' : 'complete',
+      target: 'complete',
+    },
+  ];
+  const failureCounts = new Map();
+  for (const run of eligibleRuns) {
+    if (!run.error?.code) continue;
+    failureCounts.set(run.error.code, (failureCounts.get(run.error.code) ?? 0) + 1);
+  }
+
+  return {
+    generatedAt: now,
+    ready: checks.every((check) => check.passed),
+    recommendation: checks.every((check) => check.passed)
+      ? 'manual_canary_review'
+      : 'keep_shadow',
+    window: {
+      hours: loaded.hours,
+      from: loaded.from,
+    },
+    thresholds,
+    samples: {
+      totalObservations: loaded.runs.length,
+      eligibleObservations: eligibleRuns.length,
+      excludedSynthetic: loaded.runs.length - eligibleRuns.length,
+      successes,
+      failures: eligibleRuns.length - successes,
+      successRate,
+      latencyCoverageRate,
+      latencyP95Ms,
+      nativeSettledRate,
+      distinctSessions,
+      lastObservedAt,
+      hoursSinceLastObservation,
+      sampled: loaded.capped,
+    },
+    service: {
+      status: serviceHealth?.status ?? null,
+      latencyMs: serviceHealth?.latencyMs ?? null,
+      checkpointKind: checkpoint?.kind ?? null,
+      checkpointDurable: checkpoint?.durable ?? null,
+      execution: serviceHealth?.details?.execution ?? null,
+    },
+    checks,
+    blockers: checks.filter((check) => !check.passed).map((check) => check.id),
+    failureCodes: [...failureCounts.entries()]
+      .map(([code, count]) => ({ code, count }))
+      .sort((left, right) => right.count - left.count || left.code.localeCompare(right.code))
+      .slice(0, 10),
+  };
+}
+
 function summarizeRuns(runs, { capped = false } = {}) {
   const successes = runs.filter((run) => run.shadowStatus === 'succeeded').length;
   const failures = runs.length - successes;
@@ -97,6 +275,7 @@ export function createOrchestratorObservabilityService({
   configService,
   serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
   fetchImpl = globalThis.fetch,
+  nowMs = Date.now,
 } = {}) {
   if (!pool?.query) throw new Error('Orchestrator observability requires a database pool');
   if (!configService?.getRuntimeState) {
@@ -105,7 +284,7 @@ export function createOrchestratorObservabilityService({
 
   async function loadShadowEvents({ hours = 24 } = {}) {
     const normalizedHours = clampInteger(hours, 24, 1, 24 * 30);
-    const from = Date.now() - normalizedHours * 60 * 60 * 1000;
+    const from = nowMs() - normalizedHours * 60 * 60 * 1000;
     const [rows] = await pool.query(
       `SELECT
          e.id AS event_id,
@@ -131,11 +310,19 @@ export function createOrchestratorObservabilityService({
       from,
       hours: normalizedHours,
       capped: rows.length >= MAX_METRIC_EVENTS,
-      runs: rows.map(projectShadowEventRow),
+      runs: projectUniqueShadowRuns(rows),
     };
   }
 
   return {
+    async getCanaryReadiness() {
+      const [loaded, runtimeState] = await Promise.all([
+        loadShadowEvents({ hours: CANARY_READINESS_THRESHOLDS.hours }),
+        configService.getRuntimeState({ probe: true }),
+      ]);
+      return buildCanaryReadiness(loaded, runtimeState, nowMs());
+    },
+
     async listShadowRuns({
       hours = 24,
       limit = 50,
@@ -148,7 +335,7 @@ export function createOrchestratorObservabilityService({
         ? loaded.runs
         : loaded.runs.filter((run) => run.shadowStatus === normalizedStatus);
       return {
-        generatedAt: Date.now(),
+        generatedAt: nowMs(),
         window: {
           hours: loaded.hours,
           from: loaded.from,
@@ -244,11 +431,14 @@ export function createOrchestratorObservabilityService({
 }
 
 export const orchestratorObservabilityInternals = {
+  CANARY_READINESS_THRESHOLDS,
   MAX_METRIC_EVENTS,
   SHADOW_EVENT_TYPES,
+  buildCanaryReadiness,
   normalizeRunId,
   parseJsonColumn,
   percentile,
   projectShadowEventRow,
+  projectUniqueShadowRuns,
   summarizeRuns,
 };

services/orchestrator/observability.test.mjs

@@ -1,6 +1,9 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
-import { createOrchestratorObservabilityService } from './observability.mjs';
+import {
+  createOrchestratorObservabilityService,
+  orchestratorObservabilityInternals,
+} from './observability.mjs';
 
 function response(body, status = 200) {
   return new Response(JSON.stringify(body), {
@@ -148,3 +151,145 @@ test('observability rejects unsafe run ids before querying', async () => {
   assert.equal(await service.getShadowRun('../unsafe'), null);
   assert.equal(queries, 0);
 });
+
+test('observability counts only the newest terminal event for each run', () => {
+  const base = {
+    run_id: 'same-run',
+    request_id: 'same-request',
+    user_id: 'same-user',
+    native_status: 'succeeded',
+    native_attempts: 1,
+  };
+  const runs = orchestratorObservabilityInternals.projectUniqueShadowRuns([
+    {
+      ...base,
+      event_id: 'newer-success',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 80 },
+      event_created_at: 2000,
+    },
+    {
+      ...base,
+      event_id: 'older-failure',
+      event_type: 'workflow_shadow_failed',
+      data_json: { latencyMs: 100, code: 'TIMEOUT' },
+      event_created_at: 1000,
+    },
+  ]);
+  assert.equal(runs.length, 1);
+  assert.equal(runs[0].eventId, 'newer-success');
+  assert.equal(runs[0].shadowStatus, 'succeeded');
+});
+
+test('canary readiness excludes smoke runs and reports explicit blockers', async () => {
+  const now = 10_000_000;
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        return [[{
+          event_id: 'smoke-event',
+          run_id: 'smoke-run',
+          event_type: 'workflow_shadow_completed',
+          data_json: {
+            latencyMs: 50,
+            taskType: 'orchestrator_shadow_smoke',
+          },
+          event_created_at: now - 1000,
+          request_id: 'smoke-request',
+          user_id: 'smoke-user',
+          agent_session_id: null,
+          native_status: 'succeeded',
+          native_attempts: 0,
+          native_completed_at: now,
+        }]];
+      },
+    },
+    configService: {
+      async getRuntimeState(options) {
+        assert.deepEqual(options, { probe: true });
+        return {
+          config: { mode: 'shadow' },
+          serviceHealth: {
+            ok: true,
+            status: 'healthy',
+            latencyMs: 5,
+            details: {
+              checkpoint: { kind: 'postgres', durable: true },
+              execution: 'observe-only',
+            },
+          },
+        };
+      },
+    },
+    nowMs: () => now,
+  });
+
+  const readiness = await service.getCanaryReadiness();
+  assert.equal(readiness.ready, false);
+  assert.equal(readiness.recommendation, 'keep_shadow');
+  assert.equal(readiness.samples.totalObservations, 1);
+  assert.equal(readiness.samples.eligibleObservations, 0);
+  assert.equal(readiness.samples.excludedSynthetic, 1);
+  assert.ok(readiness.blockers.includes('sample_volume'));
+  assert.ok(readiness.blockers.includes('session_coverage'));
+  assert.ok(!readiness.blockers.includes('service_healthy'));
+});
+
+test('canary readiness passes only when operational and sample gates all pass', async () => {
+  const now = 20_000_000;
+  const rows = Array.from({ length: 20 }, (_, index) => ({
+    event_id: `event-${index}`,
+    run_id: `run-${index}`,
+    event_type: index === 19
+      ? 'workflow_shadow_failed'
+      : 'workflow_shadow_completed',
+    data_json: {
+      latencyMs: 100 + index,
+      taskType: 'code_task',
+      ...(index === 19 ? { code: 'TRANSIENT', message: 'retry later' } : {}),
+    },
+    event_created_at: now - index * 1000,
+    request_id: `request-${index}`,
+    user_id: `user-${index % 3}`,
+    agent_session_id: `session-${index % 5}`,
+    native_status: index % 2 ? 'succeeded' : 'failed',
+    native_attempts: 1,
+    native_completed_at: now,
+  }));
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        return [rows];
+      },
+    },
+    configService: {
+      async getRuntimeState() {
+        return {
+          config: { mode: 'shadow' },
+          serviceHealth: {
+            ok: true,
+            status: 'healthy',
+            latencyMs: 4,
+            details: {
+              checkpoint: { kind: 'postgres', durable: true },
+              execution: 'observe-only',
+            },
+          },
+        };
+      },
+    },
+    nowMs: () => now,
+  });
+
+  const readiness = await service.getCanaryReadiness();
+  assert.equal(readiness.ready, true);
+  assert.equal(readiness.recommendation, 'manual_canary_review');
+  assert.equal(readiness.window.hours, 168);
+  assert.equal(readiness.samples.eligibleObservations, 20);
+  assert.equal(readiness.samples.successRate, 0.95);
+  assert.equal(readiness.samples.latencyCoverageRate, 1);
+  assert.equal(readiness.samples.nativeSettledRate, 1);
+  assert.equal(readiness.samples.distinctSessions, 5);
+  assert.equal(readiness.blockers.length, 0);
+  assert.deepEqual(readiness.failureCodes, [{ code: 'TRANSIENT', count: 1 }]);
+});