feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.
This commit is contained in:
john
2026-07-25 07:28:37 +08:00
parent 08a48e4849
commit 6df82818c5
33 changed files with 1569 additions and 108 deletions
+43
View File
@@ -87,6 +87,45 @@ test('LangGraph runtime is idempotent by run id', async () => {
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 3);
});
test('LangGraph runtime deletes terminal checkpoints and linked Executor Jobs', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
await runtime.start(runSpec());
assert.deepEqual(await runtime.deleteRun('run-shadow-1'), {
runId: 'run-shadow-1',
deleted: true,
executorJobDeleted: true,
});
assert.equal(await runtime.getState('run-shadow-1'), null);
assert.equal(await runtime.getExecutorJob('run-shadow-1:executor-preview'), null);
assert.equal(await runtime.deleteRun('run-shadow-1'), null);
});
test('LangGraph runtime retention purge is dry-run by default and deletes only on apply', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
await runtime.start(runSpec({ runId: 'run-retention-1' }));
const before = Date.now() + 1000;
const preview = await runtime.purgeTerminalRuns({ before });
assert.equal(preview.dryRun, true);
assert.equal(preview.candidates.length, 1);
assert.equal(await runtime.getState('run-retention-1') != null, true);
const applied = await runtime.purgeTerminalRuns({
before,
dryRun: false,
});
assert.equal(applied.deleted.length, 1);
assert.equal(applied.failures.length, 0);
assert.equal(await runtime.getState('run-retention-1'), null);
await assert.rejects(
() => runtime.purgeTerminalRuns({ before: 0 }),
(error) => error.code === 'WORKFLOW_PURGE_CUTOFF_REQUIRED',
);
});
test('LangGraph runtime persists a durable blocked Executor Job and probes both stores', async () => {
const checkpointProbeCalls = [];
const executorProbeCalls = [];
@@ -179,6 +218,10 @@ test('LangGraph active workflow waits on a leased Executor Job and converges ter
assert.equal(state.status, 'waiting');
assert.equal(state.phase, 'executor_queued');
assert.equal(state.plan.executorJob.status, 'queued');
await assert.rejects(
() => runtime.deleteRun('run-active-1'),
(error) => error.code === 'WORKFLOW_RUN_NOT_TERMINAL' && error.status === 409,
);
const claim = await runtime.claimExecutorJob({
workerId: 'worker-1',