memind/scripts/run-headroom-active-loopback.mjs

#!/usr/bin/env node
/**
 * Compare direct compat proxy vs headroom proxy on a tool-heavy request.
 * Loopback only; does not mutate goosed config unless MEMIND_HEADROOM_APPLY_GOosed=1.
 */
import { spawn } from 'node:child_process';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { prepareGooseV149CheckEnv } from './goose-v149-canary.mjs';
import {
  resolveDeepseekNoThinkProxyBaseUrl,
} from '../deepseek-no-think-proxy.mjs';
import {
  probeHeadroomProxyReachable,
  resolveGoosedApiUrlWithHeadroom,
  resolveHeadroomMode,
  resolveHeadroomProxyBaseUrl,
} from '../memind-headroom-policy.mjs';

const root = path.join(path.dirname(fileURLToPath(import.meta.url)), '..');
prepareGooseV149CheckEnv(process.env, root);

const upstreamBase = resolveDeepseekNoThinkProxyBaseUrl();
const headroomBase = resolveHeadroomProxyBaseUrl();
const model = process.env.MEMIND_HEADROOM_OBS_MODEL ?? 'deepseek-chat';
const apiKey = process.env.DEEPSEEK_API_KEY
  ?? process.env.OPENAI_API_KEY
  ?? process.env.TKMIND_EXECUTOR_API_KEY
  ?? 'local-dev';

function buildToolHeavyMessages() {
  const toolPayload = [
    'TKMind search result chunk',
    ...Array.from({ length: 400 }, (_, index) => (
      `line-${index}: portal resume smoke validates session round-trip with harness bootstrap`
    )),
  ].join('\n');

  return [
    { role: 'system', content: 'You compress long tool output into one short sentence.' },
    { role: 'user', content: 'Summarize the tool result in under 30 Chinese characters.' },
    {
      role: 'assistant',
      content: null,
      tool_calls: [{
        id: 'call_headroom_probe',
        type: 'function',
        function: { name: 'tkmind_read', arguments: '{"path":"docs/plan.md"}' },
      }],
    },
    { role: 'tool', tool_call_id: 'call_headroom_probe', content: toolPayload },
  ];
}

async function chatCompletion(baseUrl, messages, { viaHeadroom = false } = {}) {
  const headers = {
    'Content-Type': 'application/json',
    Authorization: `Bearer ${apiKey}`,
  };
  if (viaHeadroom) {
    headers['x-headroom-base-url'] = upstreamBase;
  }
  const response = await fetch(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
    method: 'POST',
    headers,
    body: JSON.stringify({
      model,
      messages,
      stream: false,
      max_tokens: 64,
      temperature: 0,
    }),
  });
  const text = await response.text();
  let payload = {};
  try {
    payload = JSON.parse(text);
  } catch {
    payload = { raw: text.slice(0, 400) };
  }
  if (!response.ok) {
    throw new Error(`${baseUrl} ${response.status}: ${text.slice(0, 400)}`);
  }
  const usage = payload?.usage ?? {};
  return {
    reply: payload?.choices?.[0]?.message?.content ?? '',
    promptTokens: Number(usage.prompt_tokens ?? usage.input_tokens ?? 0),
    completionTokens: Number(usage.completion_tokens ?? usage.output_tokens ?? 0),
    totalTokens: Number(usage.total_tokens ?? 0),
  };
}

async function ensureHeadroomProxy() {
  if (await probeHeadroomProxyReachable({ baseUrl: headroomBase })) return false;
  const port = new URL(`${headroomBase}/`).port || '8787';
  console.log(`HEADROOM_ACTIVE_START: spawning headroom proxy on :${port}`);
  const child = spawn(
    'headroom',
    ['proxy', '--port', port, '--openai-api-url', upstreamBase.replace(/\/v1\/?$/, '')],
    {
      detached: true,
      stdio: 'ignore',
      env: {
        ...process.env,
        OPENAI_TARGET_API_URL: upstreamBase,
        HEADROOM_OUTPUT_SHAPER: '0',
      },
    },
  );
  child.unref();
  for (let attempt = 0; attempt < 20; attempt += 1) {
    await new Promise((resolve) => setTimeout(resolve, 1000));
    if (await probeHeadroomProxyReachable({ baseUrl: headroomBase })) return true;
  }
  throw new Error('headroom proxy failed to become reachable');
}

async function main() {
  const previousMode = resolveHeadroomMode();
  process.env.MEMIND_HEADROOM_MODE = 'active';
  await ensureHeadroomProxy();

  const reachable = await probeHeadroomProxyReachable({ baseUrl: headroomBase });
  if (!reachable) {
    throw new Error(`headroom proxy unreachable at ${headroomBase}`);
  }

  const routing = resolveGoosedApiUrlWithHeadroom({
    apiUrl: upstreamBase,
    mode: 'active',
    headroomReachable: true,
    eligible: true,
  });
  if (!routing.routed) {
    throw new Error(`headroom routing not active: ${JSON.stringify(routing)}`);
  }

  const messages = buildToolHeavyMessages();
  const direct = await chatCompletion(upstreamBase, messages);
  const viaHeadroom = await chatCompletion(headroomBase, messages, { viaHeadroom: true });

  const savedPromptTokens = direct.promptTokens - viaHeadroom.promptTokens;
  const savedRatio = direct.promptTokens > 0
    ? savedPromptTokens / direct.promptTokens
    : 0;

  console.log('HEADROOM_ACTIVE_OBSERVATION:');
  console.log(`  mode=${previousMode}->active`);
  console.log(`  upstream=${upstreamBase}`);
  console.log(`  headroom=${headroomBase}`);
  console.log(`  model=${model}`);
  console.log(`  direct_prompt_tokens=${direct.promptTokens}`);
  console.log(`  headroom_prompt_tokens=${viaHeadroom.promptTokens}`);
  console.log(`  saved_prompt_tokens=${savedPromptTokens}`);
  console.log(`  saved_prompt_ratio=${savedRatio.toFixed(4)}`);
  console.log(`  direct_reply=${JSON.stringify(direct.reply).slice(0, 120)}`);
  console.log(`  headroom_reply=${JSON.stringify(viaHeadroom.reply).slice(0, 120)}`);

  if (viaHeadroom.promptTokens >= direct.promptTokens) {
    console.warn('HEADROOM_ACTIVE_WARN: no prompt token reduction observed on this sample');
  } else {
    console.log('HEADROOM_ACTIVE_OK: prompt tokens reduced through headroom proxy');
  }
}

main().catch((error) => {
  console.error(`HEADROOM_ACTIVE_FAIL: ${error.message}`);
  process.exit(1);
});