feat: add workflow dry-run observability

2026-07-24 22:44:26 +08:00
parent c9ad841543
commit 19706eb3a0
15 changed files with 725 additions and 41 deletions
@@ -73,6 +73,118 @@ test('observability aggregates shadow outcomes and latency percentiles', async (
  assert.equal(result.runs[1].error.code, 'TIMEOUT');
 });

+test('observability aggregates complete dry-run routing decisions and Native outcomes', async () => {
+  const rows = [
+    {
+      event_id: 'plan-event-3',
+      run_id: 'plan-run-3',
+      data_json: {
+        mode: 'canary',
+        candidateEngine: 'langgraph',
+        effectiveEngine: 'native',
+        fallbackEngine: 'native',
+        reason: 'execution_gate_disabled',
+        candidateReason: 'percentage',
+        configVersion: 4,
+        bucket: 12,
+        taskType: 'repo_refactor',
+        dryRun: true,
+        handoffAllowed: false,
+      },
+      event_created_at: 3000,
+      request_id: 'plan-request-3',
+      user_id: 'plan-user-3',
+      agent_session_id: 'plan-session-2',
+      native_status: 'failed',
+      native_attempts: 2,
+      native_completed_at: 3100,
+    },
+    {
+      event_id: 'plan-event-2',
+      run_id: 'plan-run-2',
+      data_json: JSON.stringify({
+        mode: 'canary',
+        candidateEngine: 'native',
+        effectiveEngine: 'native',
+        reason: 'canary_not_selected',
+        configVersion: 4,
+        bucket: 78,
+        taskType: 'code_analysis',
+        dryRun: true,
+        handoffAllowed: false,
+      }),
+      event_created_at: 2000,
+      request_id: 'plan-request-2',
+      user_id: 'plan-user-2',
+      agent_session_id: 'plan-session-1',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 2100,
+    },
+    {
+      event_id: 'plan-event-1',
+      run_id: 'plan-run-1',
+      data_json: {
+        mode: 'canary',
+        candidateEngine: 'langgraph',
+        effectiveEngine: 'native',
+        reason: 'execution_gate_disabled',
+        candidateReason: 'user_allowlist',
+        taskType: 'repo_refactor',
+        dryRun: true,
+        handoffAllowed: false,
+      },
+      event_created_at: 1000,
+      request_id: 'plan-request-1',
+      user_id: 'plan-user-1',
+      agent_session_id: 'plan-session-1',
+      native_status: 'running',
+      native_attempts: 1,
+      native_completed_at: null,
+    },
+  ];
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query(sql, params) {
+        assert.match(sql, /e\.event_type = \?/);
+        assert.equal(params[0], 'workflow_execution_planned');
+        return [rows];
+      },
+    },
+    configService: { getRuntimeState() {} },
+    nowMs: () => 5000,
+  });
+
+  const result = await service.listExecutionPlans({
+    hours: 12,
+    limit: 10,
+    selection: 'candidate',
+  });
+  assert.equal(result.window.hours, 12);
+  assert.equal(result.metrics.decisions, 3);
+  assert.equal(result.metrics.candidateSelections, 2);
+  assert.equal(result.metrics.candidateSelectionRate, 2 / 3);
+  assert.equal(result.metrics.nativeSelections, 1);
+  assert.equal(result.metrics.nativeSucceeded, 1);
+  assert.equal(result.metrics.nativeFailed, 1);
+  assert.equal(result.metrics.nativeSettledRate, 2 / 3);
+  assert.equal(result.metrics.handoffAllowed, 0);
+  assert.equal(result.metrics.distinctSessions, 2);
+  assert.deepEqual(result.metrics.candidateReasons, [
+    { value: 'canary_not_selected', count: 1 },
+    { value: 'percentage', count: 1 },
+    { value: 'user_allowlist', count: 1 },
+  ]);
+  assert.deepEqual(result.metrics.taskTypes, [
+    { value: 'repo_refactor', count: 2 },
+    { value: 'code_analysis', count: 1 },
+  ]);
+  assert.equal(result.plans.length, 2);
+  assert.equal(result.plans[0].candidateEngine, 'langgraph');
+  assert.equal(result.plans[0].effectiveEngine, 'native');
+  assert.equal(result.plans[0].taskType, 'repo_refactor');
+});
+
 test('observability detail joins Native state with remote LangGraph checkpoint events', async () => {
  const queries = [];
  const service = createOrchestratorObservabilityService({