feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
@@ -1,7 +1,10 @@
 import assert from 'node:assert/strict';
 import net from 'node:net';
 import test from 'node:test';
-import { startOrchestratorServer } from './server.mjs';
+import {
+  orchestratorServerInternals,
+  startOrchestratorServer,
+} from './server.mjs';

 async function reservePort() {
  const server = net.createServer();
@@ -46,3 +49,79 @@ test('orchestrator server owns checkpoint and Executor Job Store lifecycle', asy
  closed = true;
  assert.equal(running.server.listening, false);
 });
+
+test('orchestrator server fails closed for exposed or executable configurations without tokens', () => {
+  assert.throws(
+    () => orchestratorServerInternals.assertSecureServerConfig({
+      host: '0.0.0.0',
+      executionEnabled: false,
+    }),
+    (error) => error.code === 'ORCHESTRATOR_SERVICE_TOKEN_REQUIRED',
+  );
+  assert.throws(
+    () => orchestratorServerInternals.assertSecureServerConfig({
+      host: '127.0.0.1',
+      executionEnabled: true,
+      serviceToken: 'service-token',
+    }),
+    (error) => error.code === 'ORCHESTRATOR_EXECUTION_TOKENS_REQUIRED',
+  );
+  assert.throws(
+    () => orchestratorServerInternals.assertSecureServerConfig({
+      host: '127.0.0.1',
+      executionEnabled: true,
+      serviceToken: 'shared-token',
+      workerToken: 'shared-token',
+    }),
+    (error) => error.code === 'ORCHESTRATOR_EXECUTION_TOKENS_NOT_DISTINCT',
+  );
+  assert.doesNotThrow(
+    () => orchestratorServerInternals.assertSecureServerConfig({
+      host: '0.0.0.0',
+      executionEnabled: true,
+      serviceToken: 'service-token',
+      workerToken: 'worker-token',
+    }),
+  );
+});
+
+test('orchestrator retention sweep is disabled by default and applies an explicit cutoff', async () => {
+  const calls = [];
+  let intervalCallback = null;
+  let cleared = false;
+  const runtime = {
+    async purgeTerminalRuns(options) {
+      calls.push(options);
+      return { deleted: [], failures: [] };
+    },
+  };
+  assert.equal(
+    orchestratorServerInternals.startRetentionSweep(runtime, {
+      retentionDays: 0,
+    }),
+    null,
+  );
+  const sweep = orchestratorServerInternals.startRetentionSweep(runtime, {
+    retentionDays: 30,
+    intervalMs: 60 * 60 * 1000,
+    nowMs: () => 40 * 24 * 60 * 60 * 1000,
+    setIntervalFn(callback) {
+      intervalCallback = callback;
+      return { unref() {} };
+    },
+    clearIntervalFn() {
+      cleared = true;
+    },
+    logger: { log() {}, warn() {} },
+  });
+  assert.equal(sweep.retentionDays, 30);
+  intervalCallback();
+  await new Promise((resolve) => setImmediate(resolve));
+  assert.deepEqual(calls, [{
+    before: 10 * 24 * 60 * 60 * 1000,
+    limit: 100,
+    dryRun: false,
+  }]);
+  sweep.stop();
+  assert.equal(cleared, true);
+});