memind/services/orchestrator/shadow-observer.test.mjs

import assert from 'node:assert/strict';
import test from 'node:test';
import { createWorkflowShadowObserver } from './shadow-observer.mjs';

function jsonResponse(body, status = 200) {
  return new Response(JSON.stringify(body), {
    status,
    headers: { 'content-type': 'application/json' },
  });
}

test('shadow observer skips without creating a remote client when mode does not select shadow', async () => {
  let fetchCalls = 0;
  const observer = createWorkflowShadowObserver({
    configService: {
      async selectEngine() {
        return { shadowEngine: null, reason: 'mode_off', mode: 'off' };
      },
      async getRuntimeState() {
        throw new Error('must not load runtime config');
      },
    },
    fetchImpl: async () => {
      fetchCalls += 1;
      return jsonResponse({});
    },
  });

  const result = await observer({
    runId: 'run-1',
    requestId: 'request-1',
    userId: 'user-1',
  });
  assert.deepEqual(result, {
    observed: false,
    reason: 'mode_off',
    mode: 'off',
    executionPlan: null,
  });
  assert.equal(fetchCalls, 0);
});

test('shadow boundary returns an auditable dry-run plan without calling LangGraph', async () => {
  let fetchCalls = 0;
  const observer = createWorkflowShadowObserver({
    configService: {
      async selectEngine() {
        return {
          engine: 'native',
          candidateEngine: 'langgraph',
          shadowEngine: null,
          fallbackEngine: 'native',
          reason: 'execution_gate_disabled',
          candidateReason: 'user_allowlist',
          mode: 'canary',
          configVersion: 8,
          bucket: 42,
          dryRun: true,
        };
      },
      async getRuntimeState() {
        throw new Error('must not load runtime config');
      },
    },
    fetchImpl: async () => {
      fetchCalls += 1;
      return jsonResponse({});
    },
  });

  const result = await observer({
    runId: 'run-dry-1',
    requestId: 'request-dry-1',
    userId: 'user-1',
    taskType: 'repo_refactor',
  });
  assert.equal(result.observed, false);
  assert.deepEqual(result.executionPlan, {
    version: 'workflow-execution-decision-v1',
    candidateEngine: 'langgraph',
    effectiveEngine: 'native',
    fallbackEngine: 'native',
    reason: 'execution_gate_disabled',
    candidateReason: 'user_allowlist',
    configVersion: 8,
    bucket: 42,
    taskType: 'repo_refactor',
    dryRun: true,
    handoffAllowed: false,
  });
  assert.equal(fetchCalls, 0);
});

test('shadow boundary records non-selected canary decisions for a complete rate denominator', async () => {
  const observer = createWorkflowShadowObserver({
    configService: {
      async selectEngine() {
        return {
          engine: 'native',
          candidateEngine: 'native',
          shadowEngine: null,
          fallbackEngine: 'native',
          reason: 'canary_not_selected',
          candidateReason: null,
          mode: 'canary',
          configVersion: 9,
          bucket: 88,
          dryRun: false,
        };
      },
      async getRuntimeState() {
        throw new Error('must not load runtime config');
      },
    },
  });

  const result = await observer({
    runId: 'run-native-1',
    requestId: 'request-native-1',
    userId: 'user-1',
    taskType: 'code_analysis',
  });
  assert.equal(result.observed, false);
  assert.equal(result.executionPlan.dryRun, true);
  assert.equal(result.executionPlan.candidateEngine, 'native');
  assert.equal(result.executionPlan.reason, 'canary_not_selected');
  assert.equal(result.executionPlan.taskType, 'code_analysis');
});

test('shadow observer sends a bounded observe-only RunSpec to LangGraph service', async () => {
  let capturedUrl = null;
  let capturedInit = null;
  const observer = createWorkflowShadowObserver({
    configService: {
      async selectEngine() {
        return {
          shadowEngine: 'langgraph',
          reason: 'shadow',
          mode: 'shadow',
          configVersion: 7,
        };
      },
      async getRuntimeState() {
        return {
          config: {
            serviceUrl: 'http://orchestrator.internal:8093',
            requestTimeoutMs: 1200,
          },
        };
      },
    },
    serviceToken: 'internal-token',
    fetchImpl: async (url, init) => {
      capturedUrl = url;
      capturedInit = init;
      return jsonResponse({ runId: 'run-2', status: 'succeeded' }, 202);
    },
  });

  const result = await observer({
    runId: 'run-2',
    requestId: 'request-2',
    userId: 'user-2',
    sessionId: 'session-2',
    taskType: 'code-change',
    userMessage: {
      content: [{ type: 'text', text: 'Implement the service boundary' }],
    },
  });

  assert.equal(result.observed, true);
  assert.equal(capturedUrl, 'http://orchestrator.internal:8093/v1/runs');
  assert.equal(capturedInit.headers.authorization, 'Bearer internal-token');
  const body = JSON.parse(capturedInit.body);
  assert.equal(body.version, 'orchestrator-run-v1');
  assert.equal(body.policy.executionMode, 'observe-only');
  assert.equal(body.policy.sideEffectsAllowed, false);
  assert.deepEqual(body.input.sessionRef, { kind: 'goose-session', id: 'session-2' });
  assert.equal(body.metadata.configVersion, 7);
});