feat: add orchestrator shadow observability

2026-07-24 21:44:21 +08:00
parent 24336d0178
commit 3e57f85d3a
18 changed files with 992 additions and 9 deletions
@@ -0,0 +1,150 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+import { createOrchestratorObservabilityService } from './observability.mjs';
+
+function response(body, status = 200) {
+  return new Response(JSON.stringify(body), {
+    status,
+    headers: { 'content-type': 'application/json' },
+  });
+}
+
+test('observability aggregates shadow outcomes and latency percentiles', async () => {
+  const rows = [
+    {
+      event_id: 'event-3',
+      run_id: 'run-3',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 300, engine: 'langgraph', taskType: 'refactor' },
+      event_created_at: 3000,
+      request_id: 'request-3',
+      user_id: 'user-3',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 3100,
+    },
+    {
+      event_id: 'event-2',
+      run_id: 'run-2',
+      event_type: 'workflow_shadow_failed',
+      data_json: JSON.stringify({ latencyMs: 200, code: 'TIMEOUT', message: 'timed out' }),
+      event_created_at: 2000,
+      request_id: 'request-2',
+      user_id: 'user-2',
+      native_status: 'succeeded',
+      native_attempts: 1,
+      native_completed_at: 2200,
+    },
+    {
+      event_id: 'event-1',
+      run_id: 'run-1',
+      event_type: 'workflow_shadow_completed',
+      data_json: { latencyMs: 100, executorAdapter: 'native-agent-run' },
+      event_created_at: 1000,
+      request_id: 'request-1',
+      user_id: 'user-1',
+      native_status: 'failed',
+      native_attempts: 2,
+      native_completed_at: 1500,
+    },
+  ];
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query(sql) {
+        assert.match(sql, /INNER JOIN h5_agent_runs/);
+        return [rows];
+      },
+    },
+    configService: { getRuntimeState() {} },
+  });
+
+  const result = await service.listShadowRuns({ hours: 12, limit: 2 });
+  assert.equal(result.window.hours, 12);
+  assert.equal(result.metrics.observations, 3);
+  assert.equal(result.metrics.successes, 2);
+  assert.equal(result.metrics.failures, 1);
+  assert.equal(result.metrics.latencyP50Ms, 200);
+  assert.equal(result.metrics.latencyP95Ms, 300);
+  assert.equal(result.metrics.nativeSucceeded, 2);
+  assert.equal(result.runs.length, 2);
+  assert.equal(result.runs[1].error.code, 'TIMEOUT');
+});
+
+test('observability detail joins Native state with remote LangGraph checkpoint events', async () => {
+  const queries = [];
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query(sql, params) {
+        queries.push({ sql, params });
+        if (sql.includes('FROM h5_agent_runs')) {
+          return [[{
+            id: 'run-detail',
+            request_id: 'request-detail',
+            user_id: 'user-detail',
+            agent_session_id: 'session-detail',
+            status: 'succeeded',
+            attempts: 1,
+            error_message: null,
+            created_at: 100,
+            updated_at: 300,
+            started_at: 150,
+            completed_at: 300,
+          }]];
+        }
+        return [[{
+          event_id: 'local-event',
+          event_type: 'workflow_shadow_completed',
+          data_json: { latencyMs: 25 },
+          created_at: 250,
+        }]];
+      },
+    },
+    configService: {
+      async getRuntimeState() {
+        return {
+          config: {
+            serviceUrl: 'http://orchestrator.internal:8093',
+            requestTimeoutMs: 1000,
+          },
+        };
+      },
+    },
+    serviceToken: 'detail-token',
+    fetchImpl: async (url, init) => {
+      assert.equal(init.headers.authorization, 'Bearer detail-token');
+      if (String(url).endsWith('/events')) {
+        return response({
+          events: [{ sequence: 1, type: 'workflow_validated' }],
+        });
+      }
+      return response({
+        runId: 'run-detail',
+        status: 'succeeded',
+        plan: { executorAdapter: 'native-agent-run' },
+      });
+    },
+  });
+
+  const detail = await service.getShadowRun('run-detail');
+  assert.equal(detail.native.status, 'succeeded');
+  assert.equal(detail.localEvents[0].data.latencyMs, 25);
+  assert.equal(detail.remote.available, true);
+  assert.equal(detail.remote.state.plan.executorAdapter, 'native-agent-run');
+  assert.equal(detail.remote.events[0].type, 'workflow_validated');
+  assert.equal(queries.length, 2);
+});
+
+test('observability rejects unsafe run ids before querying', async () => {
+  let queries = 0;
+  const service = createOrchestratorObservabilityService({
+    pool: {
+      async query() {
+        queries += 1;
+        return [[]];
+      },
+    },
+    configService: { getRuntimeState() {} },
+  });
+  assert.equal(await service.getShadowRun('../unsafe'), null);
+  assert.equal(queries, 0);
+});