feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.

services/orchestrator/shadow-observer.test.mjs

@@ -1,5 +1,8 @@
 import assert from 'node:assert/strict';
 import test from 'node:test';
+import { MemorySaver } from '@langchain/langgraph';
+import { createOrchestratorApp } from './app.mjs';
+import { createLangGraphOrchestratorRuntime } from './runtime.mjs';
 import { createWorkflowShadowObserver } from './shadow-observer.mjs';
 
 function jsonResponse(body, status = 200) {
@@ -9,6 +12,19 @@ function jsonResponse(body, status = 200) {
   });
 }
 
+async function listen(app) {
+  const server = await new Promise((resolve) => {
+    const candidate = app.listen(0, '127.0.0.1', () => resolve(candidate));
+  });
+  const address = server.address();
+  return {
+    baseUrl: `http://127.0.0.1:${address.port}`,
+    close: () => new Promise((resolve, reject) => {
+      server.close((error) => (error ? reject(error) : resolve()));
+    }),
+  };
+}
+
 test('shadow observer skips without creating a remote client when mode does not select shadow', async () => {
   let fetchCalls = 0;
   const observer = createWorkflowShadowObserver({
@@ -127,7 +143,7 @@ test('shadow boundary records non-selected canary decisions for a complete rate
   assert.equal(result.executionPlan.taskType, 'code_analysis');
 });
 
-test('shadow observer sends a bounded observe-only RunSpec to LangGraph service', async () => {
+test('shadow observer sends a control-plane-only RunSpec without user content or identity', async () => {
   let capturedUrl = null;
   let capturedInit = null;
   const observer = createWorkflowShadowObserver({
@@ -175,6 +191,98 @@ test('shadow observer sends a bounded observe-only RunSpec to LangGraph service'
   assert.equal(body.version, 'orchestrator-run-v1');
   assert.equal(body.policy.executionMode, 'observe-only');
   assert.equal(body.policy.sideEffectsAllowed, false);
-  assert.deepEqual(body.input.sessionRef, { kind: 'goose-session', id: 'session-2' });
+  assert.equal(body.input.instruction, '[shadow-control-plane-only]');
+  assert.equal('sessionRef' in body.input, false);
+  assert.deepEqual(body.subject, { tenantId: null, userId: null });
   assert.equal(body.metadata.configVersion, 7);
+  assert.equal(body.metadata.dataPolicy, 'control-plane-only-v1');
+  assert.equal(
+    capturedInit.body.includes('Implement the service boundary'),
+    false,
+  );
+  assert.equal(capturedInit.body.includes('user-2'), false);
+  assert.equal(capturedInit.body.includes('session-2'), false);
+});
+
+test('isolated Portal shadow flow reaches LangGraph over HTTP and removes all terminal state', async (t) => {
+  const checkpointer = new MemorySaver();
+  const runtime = createLangGraphOrchestratorRuntime({
+    checkpointer,
+    checkpointKind: 'memory',
+    durable: false,
+  });
+  const server = await listen(createOrchestratorApp({
+    runtime,
+    serviceToken: 'isolated-shadow-token',
+  }));
+  t.after(server.close);
+
+  const observer = createWorkflowShadowObserver({
+    configService: {
+      async selectEngine() {
+        return {
+          shadowEngine: 'langgraph',
+          reason: 'shadow',
+          mode: 'shadow',
+          configVersion: 11,
+        };
+      },
+      async getRuntimeState() {
+        return {
+          config: {
+            serviceUrl: server.baseUrl,
+            requestTimeoutMs: 2000,
+          },
+        };
+      },
+    },
+    serviceToken: 'isolated-shadow-token',
+  });
+
+  const secretUserContent = 'private user content must never cross the shadow boundary';
+  const result = await observer({
+    runId: 'isolated-shadow-run-1',
+    requestId: 'isolated-shadow-request-1',
+    userId: 'private-user-id',
+    sessionId: 'private-session-id',
+    taskType: 'repo_refactor',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: secretUserContent }],
+    },
+  });
+
+  assert.equal(result.observed, true);
+  assert.equal(result.shadowRun.status, 'succeeded');
+  assert.equal(result.shadowRun.result.executed, false);
+  const tuple = await checkpointer.getTuple({
+    configurable: {
+      thread_id: 'isolated-shadow-run-1',
+      checkpoint_ns: '',
+    },
+  });
+  assert.equal(
+    tuple.checkpoint.channel_values.spec.input.instruction,
+    '[shadow-control-plane-only]',
+  );
+  const checkpointJson = JSON.stringify(tuple.checkpoint.channel_values.spec);
+  assert.equal(checkpointJson.includes(secretUserContent), false);
+  assert.equal(checkpointJson.includes('private-user-id'), false);
+  assert.equal(checkpointJson.includes('private-session-id'), false);
+
+  const deleted = await fetch(`${server.baseUrl}/v1/runs/isolated-shadow-run-1`, {
+    method: 'DELETE',
+    headers: { authorization: 'Bearer isolated-shadow-token' },
+  });
+  assert.equal(deleted.status, 200);
+  assert.deepEqual(await deleted.json(), {
+    runId: 'isolated-shadow-run-1',
+    deleted: true,
+    executorJobDeleted: true,
+  });
+  assert.equal(await runtime.getState('isolated-shadow-run-1'), null);
+  assert.equal(
+    await runtime.getExecutorJob('isolated-shadow-run-1:executor-preview'),
+    null,
+  );
 });