feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
This commit is contained in:
john
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
+20 -6
View File
@@ -285,12 +285,18 @@ export function bootstrapPortalGatewayServices({
});
const validateRunDeliverables =
createRunDeliverablesValidatorFn({ h5Root });
const workflowShadowObserver =
createWorkflowShadowObserverFn({
configService:
createOrchestratorAdminConfigServiceFn(pool),
logger: console,
});
const workflowShadowEnabled = isEnabledFlag(
env.MEMIND_ORCHESTRATOR_SHADOW_OBSERVATION_ENABLED,
);
const workflowShadowObserver = workflowShadowEnabled
? createWorkflowShadowObserverFn({
configService:
createOrchestratorAdminConfigServiceFn(pool),
serviceToken:
env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
logger: console,
})
: null;
let runtimeRoot;
try {
runtimeRoot = fs.realpathSync(h5Root);
@@ -389,6 +395,14 @@ export function bootstrapPortalGatewayServices({
env.MEMIND_AGENT_RUN_TIMEOUT_MS ??
15 * 60 * 1000,
),
maxConcurrentShadowObservations: Number(
env.MEMIND_ORCHESTRATOR_SHADOW_MAX_CONCURRENCY ??
2,
),
maxQueuedShadowObservations: Number(
env.MEMIND_ORCHESTRATOR_SHADOW_MAX_QUEUE ??
100,
),
workerIdentity,
});
const agentRunRecoveryTimer =
@@ -166,6 +166,39 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
assert.equal(captured.agentOptions.autoDispatch, true);
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
assert.equal(captured.agentOptions.observeWorkflowRun, null);
assert.equal(captured.agentOptions.maxConcurrentShadowObservations, 2);
assert.equal(captured.agentOptions.maxQueuedShadowObservations, 100);
});
test('wires Shadow observation only behind the explicit environment gate', () => {
const configService = { id: 'orchestrator-config' };
const observer = async () => ({ observed: false });
let receivedObserverOptions = null;
const setup = createSetup({
env: {
MEMIND_ORCHESTRATOR_SHADOW_OBSERVATION_ENABLED: '1',
MEMIND_ORCHESTRATOR_SERVICE_TOKEN: 'shadow-token',
MEMIND_ORCHESTRATOR_SHADOW_MAX_CONCURRENCY: '4',
MEMIND_ORCHESTRATOR_SHADOW_MAX_QUEUE: '25',
},
createOrchestratorAdminConfigServiceFn(receivedPool) {
assert.equal(receivedPool, setup.pool);
return configService;
},
createWorkflowShadowObserverFn(options) {
receivedObserverOptions = options;
return observer;
},
});
bootstrapPortalGatewayServices(setup.options);
const { agentOptions } = setup.getCaptured();
assert.equal(receivedObserverOptions.configService, configService);
assert.equal(receivedObserverOptions.serviceToken, 'shadow-token');
assert.equal(agentOptions.observeWorkflowRun, observer);
assert.equal(agentOptions.maxConcurrentShadowObservations, 4);
assert.equal(agentOptions.maxQueuedShadowObservations, 25);
});
test('preserves local asset reads and optional asset absence', async () => {