feat(orchestrator): observe Page Data validation

2026-07-25 14:16:57 +08:00
parent e02495a6c0
commit 417aa9c78b
17 changed files with 784 additions and 16 deletions
@@ -462,6 +462,153 @@ test('agent run creation stores code tool metadata in the queued message', async
  });
 });

+test('explicit Page Data selection enters Orchestrator Shadow even when the run starts in chat mode', async () => {
+  const pool = createFakePool();
+  const observed = [];
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {},
+    autoDispatch: false,
+    observeWorkflowRun: async (input) => {
+      observed.push(input);
+      return { observed: false, reason: 'test-only' };
+    },
+  });
+
+  const run = await gateway.createRun('user-page-data-shadow', {
+    requestId: 'req-page-data-shadow',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: '解释一下这个数据空间' }],
+      metadata: {
+        displayText: '解释一下这个数据空间',
+        memindRun: { pgRequired: true },
+      },
+    },
+    toolMode: 'chat',
+  });
+
+  await waitFor(() => observed.length === 1);
+  assert.equal(observed[0].runId, run.id);
+  assert.equal(observed[0].pageDataRequired, true);
+  assert.equal(observed[0].taskType, 'page_data_dev');
+  assert.equal(pool.runs.get(run.id).status, 'queued');
+});
+
+test('successful Page Data runs send bounded validation evidence without changing Native execution', async () => {
+  const pool = createFakePool();
+  const validationInputs = [];
+  const submitted = [];
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {
+      async startSessionForUser() {
+        return { id: 'session-page-data-validation' };
+      },
+      async submitSessionReplyForUser(userId, sessionId, requestId, userMessage) {
+        submitted.push({ userId, sessionId, requestId, userMessage });
+      },
+    },
+    observeWorkflowRun: async () => ({ observed: true }),
+    observeWorkflowValidation: async (input) => {
+      validationInputs.push(input);
+      return {
+        observed: true,
+        engine: 'langgraph',
+        mode: 'shadow',
+        validation: {
+          kind: 'page-data-delivery',
+          verdict: 'inconclusive',
+        },
+      };
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-page-data-validation', {
+    requestId: 'req-page-data-validation',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: '解释 Page Data API 的用途' }],
+      metadata: {
+        displayText: '解释 Page Data API 的用途',
+        memindRun: { pgRequired: true },
+      },
+    },
+    toolMode: 'chat',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  await waitFor(() => validationInputs.length === 1);
+  assert.equal(submitted.length, 1);
+  assert.equal(validationInputs[0].runId, run.id);
+  assert.equal(validationInputs[0].observation.required, true);
+  assert.deepEqual(
+    validationInputs[0].observation.checks.map((check) => [check.id, check.status]),
+    [
+      ['agent_run_completion', 'passed'],
+      ['page_data_binding', 'skipped'],
+      ['page_data_storage_policy', 'skipped'],
+      ['page_data_deliverable', 'skipped'],
+      ['independent_review', 'skipped'],
+    ],
+  );
+  const projected = JSON.stringify(validationInputs[0].observation);
+  assert.equal(projected.includes('解释 Page Data API'), false);
+  assert.equal(pool.runs.get(run.id).status, 'succeeded');
+  await waitFor(() => pool.events.some(
+    (event) => event.runId === run.id
+      && event.eventType === 'workflow_validation_observation_completed',
+  ));
+});
+
+test('Page Data validation observation failure stays isolated from the Native result', async () => {
+  const pool = createFakePool();
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {
+      async startSessionForUser() {
+        return { id: 'session-page-data-validation-failure' };
+      },
+      async submitSessionReplyForUser() {},
+    },
+    observeWorkflowRun: async () => ({ observed: true }),
+    observeWorkflowValidation: async () => {
+      throw Object.assign(new Error('validation observer unavailable'), {
+        code: 'VALIDATION_OBSERVER_UNAVAILABLE',
+      });
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-page-data-validation-failure', {
+    requestId: 'req-page-data-validation-failure',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: '解释 Page Data API' }],
+      metadata: {
+        displayText: '解释 Page Data API',
+        memindRun: { pgRequired: true },
+      },
+    },
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  await waitFor(() => pool.events.some(
+    (event) => event.runId === run.id
+      && event.eventType === 'workflow_validation_observation_failed',
+  ));
+  assert.equal(pool.runs.get(run.id).status, 'succeeded');
+  const failure = JSON.parse(pool.events.find(
+    (event) => event.runId === run.id
+      && event.eventType === 'workflow_validation_observation_failed',
+  ).dataJson);
+  assert.equal(failure.code, 'VALIDATION_OBSERVER_UNAVAILABLE');
+});
+
 test('agent run starts a session and marks submitted reply as succeeded', async () => {
  const pool = createFakePool();
  const submitted = [];