feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
@@ -25,10 +25,112 @@ function csvList(value) {
  )];
 }

+function isLoopbackHost(value) {
+  const host = String(value ?? '').trim().toLowerCase();
+  return host === 'localhost'
+    || host === '::1'
+    || host === '[::1]'
+    || /^127(?:\.\d{1,3}){3}$/.test(host);
+}
+
+function assertSecureServerConfig({
+  host,
+  executionEnabled,
+  serviceToken,
+  workerToken,
+} = {}) {
+  const hasServiceToken = Boolean(String(serviceToken ?? '').trim());
+  const hasWorkerToken = Boolean(String(workerToken ?? '').trim());
+  if (!isLoopbackHost(host) && !hasServiceToken) {
+    const error = new Error(
+      'MEMIND_ORCHESTRATOR_SERVICE_TOKEN is required for non-loopback binding',
+    );
+    error.code = 'ORCHESTRATOR_SERVICE_TOKEN_REQUIRED';
+    throw error;
+  }
+  if (executionEnabled && (!hasServiceToken || !hasWorkerToken)) {
+    const error = new Error(
+      'Execution requires both MEMIND_ORCHESTRATOR_SERVICE_TOKEN and MEMIND_ORCHESTRATOR_WORKER_TOKEN',
+    );
+    error.code = 'ORCHESTRATOR_EXECUTION_TOKENS_REQUIRED';
+    throw error;
+  }
+  if (
+    executionEnabled
+    && String(serviceToken).trim() === String(workerToken).trim()
+  ) {
+    const error = new Error('Execution service and worker tokens must be distinct');
+    error.code = 'ORCHESTRATOR_EXECUTION_TOKENS_NOT_DISTINCT';
+    throw error;
+  }
+}
+
+function startRetentionSweep(runtime, {
+  retentionDays,
+  intervalMs = 24 * 60 * 60 * 1000,
+  limit = 100,
+  nowMs = Date.now,
+  setIntervalFn = setInterval,
+  clearIntervalFn = clearInterval,
+  logger = console,
+} = {}) {
+  const days = Number(retentionDays);
+  if (!Number.isFinite(days) || days <= 0) return null;
+  const boundedDays = Math.min(3650, Math.max(1, Math.floor(days)));
+  const configuredInterval = Number(intervalMs);
+  const boundedInterval = Number.isFinite(configuredInterval)
+    ? Math.max(60 * 60 * 1000, configuredInterval)
+    : 24 * 60 * 60 * 1000;
+  let active = false;
+  const sweep = async () => {
+    if (active) return null;
+    active = true;
+    try {
+      const result = await runtime.purgeTerminalRuns({
+        before: nowMs() - boundedDays * 24 * 60 * 60 * 1000,
+        limit,
+        dryRun: false,
+      });
+      if (result.deleted.length || result.failures.length) {
+        logger.log(
+          `[orchestrator-retention] deleted=${result.deleted.length} failures=${result.failures.length}`,
+        );
+      }
+      return result;
+    } catch (error) {
+      logger.warn(
+        '[orchestrator-retention] sweep failed:',
+        error instanceof Error ? error.message : error,
+      );
+      return null;
+    } finally {
+      active = false;
+    }
+  };
+  const timer = setIntervalFn(() => void sweep(), boundedInterval);
+  timer?.unref?.();
+  return {
+    retentionDays: boundedDays,
+    intervalMs: boundedInterval,
+    sweep,
+    stop() {
+      clearIntervalFn(timer);
+    },
+  };
+}
+
 export async function startOrchestratorServer({
  env = process.env,
  logger = console,
 } = {}) {
+  const host = String(env.MEMIND_ORCHESTRATOR_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
+  const executionEnabled = envFlag(env.MEMIND_ORCHESTRATOR_EXECUTION_ENABLED, false);
+  assertSecureServerConfig({
+    host,
+    executionEnabled,
+    serviceToken: env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
+    workerToken: env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
+  });
  const checkpoint = await createOrchestratorCheckpoint({
    mode: env.MEMIND_ORCHESTRATOR_CHECKPOINT_MODE,
    connectionString: env.MEMIND_ORCHESTRATOR_DATABASE_URL,
@@ -52,7 +154,7 @@ export async function startOrchestratorServer({
    checkpointProbe: checkpoint.probe,
    executorJobStore: executorJobs.store,
    executorJobProbe: executorJobs.probe,
-    executionEnabled: envFlag(env.MEMIND_ORCHESTRATOR_EXECUTION_ENABLED, false),
+    executionEnabled,
    enabledExecutors: csvList(env.MEMIND_ORCHESTRATOR_ENABLED_EXECUTORS),
    admissionConfig: {
      tenantAllowlist: csvList(env.MEMIND_ORCHESTRATOR_TENANT_ALLOWLIST),
@@ -72,8 +174,8 @@ export async function startOrchestratorServer({
    serviceToken: env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
    workerToken: env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
  });
-  const host = String(env.MEMIND_ORCHESTRATOR_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
  const port = positivePort(env.MEMIND_ORCHESTRATOR_PORT, 8093);
+  let retentionSweep = null;
  let server;
  try {
    server = await new Promise((resolve, reject) => {
@@ -84,9 +186,16 @@ export async function startOrchestratorServer({
    await Promise.allSettled([executorJobs.close(), checkpoint.close()]);
    throw error;
  }
+  retentionSweep = startRetentionSweep(runtime, {
+    retentionDays: env.MEMIND_ORCHESTRATOR_RETENTION_DAYS,
+    intervalMs: env.MEMIND_ORCHESTRATOR_RETENTION_SWEEP_INTERVAL_MS,
+    limit: env.MEMIND_ORCHESTRATOR_RETENTION_SWEEP_LIMIT,
+    logger,
+  });
  logger.log(`[orchestrator] listening on http://${host}:${port} (${checkpoint.kind})`);

  async function close() {
+    retentionSweep?.stop();
    let serverCloseError = null;
    try {
      await new Promise((resolve, reject) => {
@@ -110,6 +219,7 @@ export async function startOrchestratorServer({
    runtime,
    checkpoint,
    executorJobs,
+    retentionSweep,
    close,
  };
 }
@@ -129,7 +239,10 @@ if (isEntrypoint) {
 }

 export const orchestratorServerInternals = {
+  assertSecureServerConfig,
  csvList,
  envFlag,
+  isLoopbackHost,
  positivePort,
+  startRetentionSweep,
 };