feat(orchestrator): harden zero-impact shadow rollout
Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
This commit is contained in:
@@ -285,12 +285,18 @@ export function bootstrapPortalGatewayServices({
|
||||
});
|
||||
const validateRunDeliverables =
|
||||
createRunDeliverablesValidatorFn({ h5Root });
|
||||
const workflowShadowObserver =
|
||||
createWorkflowShadowObserverFn({
|
||||
configService:
|
||||
createOrchestratorAdminConfigServiceFn(pool),
|
||||
logger: console,
|
||||
});
|
||||
const workflowShadowEnabled = isEnabledFlag(
|
||||
env.MEMIND_ORCHESTRATOR_SHADOW_OBSERVATION_ENABLED,
|
||||
);
|
||||
const workflowShadowObserver = workflowShadowEnabled
|
||||
? createWorkflowShadowObserverFn({
|
||||
configService:
|
||||
createOrchestratorAdminConfigServiceFn(pool),
|
||||
serviceToken:
|
||||
env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
||||
logger: console,
|
||||
})
|
||||
: null;
|
||||
let runtimeRoot;
|
||||
try {
|
||||
runtimeRoot = fs.realpathSync(h5Root);
|
||||
@@ -389,6 +395,14 @@ export function bootstrapPortalGatewayServices({
|
||||
env.MEMIND_AGENT_RUN_TIMEOUT_MS ??
|
||||
15 * 60 * 1000,
|
||||
),
|
||||
maxConcurrentShadowObservations: Number(
|
||||
env.MEMIND_ORCHESTRATOR_SHADOW_MAX_CONCURRENCY ??
|
||||
2,
|
||||
),
|
||||
maxQueuedShadowObservations: Number(
|
||||
env.MEMIND_ORCHESTRATOR_SHADOW_MAX_QUEUE ??
|
||||
100,
|
||||
),
|
||||
workerIdentity,
|
||||
});
|
||||
const agentRunRecoveryTimer =
|
||||
|
||||
@@ -166,6 +166,39 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
|
||||
assert.equal(captured.agentOptions.autoDispatch, true);
|
||||
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
||||
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
||||
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
||||
assert.equal(captured.agentOptions.maxConcurrentShadowObservations, 2);
|
||||
assert.equal(captured.agentOptions.maxQueuedShadowObservations, 100);
|
||||
});
|
||||
|
||||
test('wires Shadow observation only behind the explicit environment gate', () => {
|
||||
const configService = { id: 'orchestrator-config' };
|
||||
const observer = async () => ({ observed: false });
|
||||
let receivedObserverOptions = null;
|
||||
const setup = createSetup({
|
||||
env: {
|
||||
MEMIND_ORCHESTRATOR_SHADOW_OBSERVATION_ENABLED: '1',
|
||||
MEMIND_ORCHESTRATOR_SERVICE_TOKEN: 'shadow-token',
|
||||
MEMIND_ORCHESTRATOR_SHADOW_MAX_CONCURRENCY: '4',
|
||||
MEMIND_ORCHESTRATOR_SHADOW_MAX_QUEUE: '25',
|
||||
},
|
||||
createOrchestratorAdminConfigServiceFn(receivedPool) {
|
||||
assert.equal(receivedPool, setup.pool);
|
||||
return configService;
|
||||
},
|
||||
createWorkflowShadowObserverFn(options) {
|
||||
receivedObserverOptions = options;
|
||||
return observer;
|
||||
},
|
||||
});
|
||||
|
||||
bootstrapPortalGatewayServices(setup.options);
|
||||
const { agentOptions } = setup.getCaptured();
|
||||
assert.equal(receivedObserverOptions.configService, configService);
|
||||
assert.equal(receivedObserverOptions.serviceToken, 'shadow-token');
|
||||
assert.equal(agentOptions.observeWorkflowRun, observer);
|
||||
assert.equal(agentOptions.maxConcurrentShadowObservations, 4);
|
||||
assert.equal(agentOptions.maxQueuedShadowObservations, 25);
|
||||
});
|
||||
|
||||
test('preserves local asset reads and optional asset absence', async () => {
|
||||
|
||||
Reference in New Issue
Block a user