feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
@@ -457,6 +457,105 @@ test('agent run starts a session and marks submitted reply as succeeded', async
  assert.equal(pool.runs.get(run.id).attempts, 1);
 });

+test('workflow shadow failure cannot change the native agent run result', async () => {
+  const pool = createFakePool();
+  const submitted = [];
+  const shadowError = Object.assign(new Error('orchestrator unavailable'), {
+    code: 'ORCHESTRATOR_UNAVAILABLE',
+  });
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {
+      async startSessionForUser() {
+        return { id: 'session-shadow-failure' };
+      },
+      async submitSessionReplyForUser(userId, sessionId, requestId, userMessage) {
+        submitted.push({ userId, sessionId, requestId, userMessage });
+      },
+    },
+    observeWorkflowRun: async () => {
+      throw shadowError;
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-shadow-failure', {
+    requestId: 'req-shadow-failure',
+    userMessage: { role: 'user', content: [{ type: 'text', text: 'native must win' }] },
+    toolMode: 'code',
+    taskType: 'repo_refactor',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  await waitFor(() => pool.events.some(
+    (event) => event.runId === run.id && event.eventType === 'workflow_shadow_failed',
+  ));
+
+  assert.equal(submitted.length, 1);
+  assert.equal(pool.runs.get(run.id).status, 'succeeded');
+  assert.equal(pool.runs.get(run.id).attempts, 1);
+  const failureEvent = pool.events.find(
+    (event) => event.runId === run.id && event.eventType === 'workflow_shadow_failed',
+  );
+  const failureData = JSON.parse(failureEvent.dataJson);
+  assert.equal(failureData.code, 'ORCHESTRATOR_UNAVAILABLE');
+  assert.equal(failureData.message, 'orchestrator unavailable');
+  assert.ok(Number.isFinite(failureData.latencyMs));
+  assert.ok(failureData.latencyMs >= 0);
+});
+
+test('workflow shadow queue overflow skips observation without changing native queued runs', async () => {
+  const pool = createFakePool();
+  const observedRunIds = [];
+  let releaseFirstObservation;
+  const firstObservationBlocked = new Promise((resolve) => {
+    releaseFirstObservation = resolve;
+  });
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {},
+    autoDispatch: false,
+    maxConcurrentShadowObservations: 1,
+    maxQueuedShadowObservations: 1,
+    observeWorkflowRun: async (input) => {
+      observedRunIds.push(input.runId);
+      if (observedRunIds.length === 1) await firstObservationBlocked;
+      return { observed: false, reason: 'test_observation_complete' };
+    },
+  });
+
+  const createCodeRun = (suffix) => gateway.createRun(`user-shadow-${suffix}`, {
+    requestId: `req-shadow-${suffix}`,
+    userMessage: { role: 'user', content: [{ type: 'text', text: `run ${suffix}` }] },
+    toolMode: 'code',
+    taskType: 'repo_refactor',
+  });
+
+  const first = await createCodeRun('first');
+  await waitFor(() => observedRunIds.length === 1);
+  const second = await createCodeRun('second');
+  const third = await createCodeRun('third');
+
+  await waitFor(() => pool.events.some(
+    (event) => event.runId === third.id && event.eventType === 'workflow_shadow_skipped',
+  ));
+  assert.deepEqual(
+    [first, second, third].map((run) => pool.runs.get(run.id).status),
+    ['queued', 'queued', 'queued'],
+  );
+  assert.deepEqual(JSON.parse(pool.events.find(
+    (event) => event.runId === third.id && event.eventType === 'workflow_shadow_skipped',
+  ).dataJson), {
+    reason: 'shadow_queue_full',
+  });
+
+  releaseFirstObservation();
+  await waitFor(() => observedRunIds.length === 2);
+  assert.deepEqual(observedRunIds, [first.id, second.id]);
+});
+
 test('agent run policy allow path preserves existing routing and submission behavior', async () => {
  const pool = createFakePool();
  const submitted = [];