feat: add dry-run workflow execution boundary

This commit is contained in:
john
2026-07-24 22:30:41 +08:00
parent 646178944d
commit c9ad841543
12 changed files with 553 additions and 17 deletions
+19 -4
View File
@@ -72,9 +72,22 @@ window and never changes the runtime mode. The first gate requires:
Passing the gate means only `manual_canary_review`; it does not authorize or
activate Canary routing.
Phase 3 adds the execution handoff boundary without enabling it:
- `workflow-execution-request-v1` normalizes authorization, timeout,
cancellation, fallback, and idempotency controls;
- `workflow-execution-decision-v1` separates the candidate engine from the
effective executor;
- Canary and Active selections emit auditable `workflow_execution_planned`
events;
- the effective engine remains Native and non-Native dispatch throws
`WORKFLOW_EXECUTION_HANDOFF_DISABLED`;
- the implementation capability is hard-disabled in code, so memindadm
configuration cannot accidentally unlock execution.
The default memindadm mode remains `off`. Canary and Active routing are not
wired to the LangGraph executor in Phase 2. Native Agent Run remains the sole
executor, including in Shadow mode.
wired to the LangGraph executor in Phase 3. Native Agent Run remains the sole
executor in every mode.
## Admin configuration
@@ -84,8 +97,10 @@ Supported modes:
- `off`: Native Agent Run only.
- `shadow`: Native executes; the configured engine may observe.
- `canary`: user allowlist and deterministic rollout percentage.
- `active`: the primary engine handles workflow-allowlisted tasks.
- `canary`: user allowlist and deterministic rollout percentage produce
Dry-run candidate decisions.
- `active`: workflow-allowlisted tasks produce Dry-run primary-engine
decisions.
`MEMIND_ORCHESTRATOR_KILL_SWITCH=1` always forces Native selection.
+60 -7
View File
@@ -10,6 +10,7 @@ import {
const CONFIG_TABLE = 'h5_orchestrator_admin_config';
const CONFIG_SCOPE = 'global';
const EXECUTION_HANDOFF_IMPLEMENTED = false;
function normalizeBoolean(value, fallback = false) {
if (value == null || value === '') return fallback;
@@ -113,17 +114,42 @@ function runtimeState(config, env = process.env) {
if (killSwitch) reason = 'kill_switch';
else if (config.mode === ORCHESTRATOR_MODE.OFF) reason = 'mode_off';
else if (!configured) reason = 'service_url_missing';
const executionModeSelected = [ORCHESTRATOR_MODE.CANARY, ORCHESTRATOR_MODE.ACTIVE]
.includes(config.mode);
const executionHandoffRequested = executionModeSelected
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH;
const executionHandoffEnabled = !reason
&& EXECUTION_HANDOFF_IMPLEMENTED
&& executionHandoffRequested
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH;
return {
killSwitch,
configured,
effective: !reason,
reason,
executesLangGraph: !reason
&& [ORCHESTRATOR_MODE.CANARY, ORCHESTRATOR_MODE.ACTIVE].includes(config.mode)
reason: reason ?? (
executionModeSelected
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH
&& !executionHandoffEnabled
? 'execution_gate_disabled'
: null
),
executesLangGraph: executionHandoffEnabled,
plansLangGraph: !reason
&& executionModeSelected
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH,
shadowsLangGraph: !reason
&& config.mode === ORCHESTRATOR_MODE.SHADOW
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH,
executionHandoff: {
implemented: EXECUTION_HANDOFF_IMPLEMENTED,
requested: executionHandoffRequested,
enabled: executionHandoffEnabled,
reason: executionHandoffEnabled
? null
: EXECUTION_HANDOFF_IMPLEMENTED
? 'execution_handoff_not_requested'
: 'phase3_dry_run_only',
},
};
}
@@ -324,12 +350,15 @@ export function createOrchestratorAdminConfigService(pool, {
const workflowMatched = config.workflowAllowlist.includes(normalizedWorkflow);
const base = {
engine: WORKFLOW_ENGINE.NATIVE,
candidateEngine: WORKFLOW_ENGINE.NATIVE,
shadowEngine: null,
fallbackEngine: config.fallbackToNative ? config.fallbackEngine : null,
mode: config.mode,
workflowMatched,
configVersion: state.configVersion,
reason: runtime.reason,
candidateReason: null,
dryRun: false,
};
if (!runtime.effective || !workflowMatched) {
return { ...base, reason: runtime.reason ?? 'workflow_not_enabled' };
@@ -342,16 +371,39 @@ export function createOrchestratorAdminConfigService(pool, {
};
}
if (config.mode === ORCHESTRATOR_MODE.ACTIVE) {
return { ...base, engine: config.primaryEngine, reason: 'active' };
const selected = {
...base,
candidateEngine: config.primaryEngine,
candidateReason: 'active',
};
if (config.primaryEngine === WORKFLOW_ENGINE.NATIVE) {
return { ...selected, reason: 'active' };
}
return runtime.executionHandoff.enabled
? { ...selected, engine: config.primaryEngine, reason: 'active' }
: { ...selected, reason: 'execution_gate_disabled', dryRun: true };
}
if (config.mode === ORCHESTRATOR_MODE.CANARY) {
const explicitlyAllowed = Boolean(userId && config.userAllowlist.includes(String(userId)));
const rolloutKey = runId || requestId || `${userId ?? ''}:${normalizedWorkflow}`;
const bucket = stableRolloutBucket(rolloutKey);
const percentAllowed = bucket < config.rolloutPercent;
return explicitlyAllowed || percentAllowed
? { ...base, engine: config.primaryEngine, reason: explicitlyAllowed ? 'user_allowlist' : 'percentage', bucket }
: { ...base, reason: 'canary_not_selected', bucket };
if (!explicitlyAllowed && !percentAllowed) {
return { ...base, reason: 'canary_not_selected', bucket };
}
const candidateReason = explicitlyAllowed ? 'user_allowlist' : 'percentage';
const selected = {
...base,
candidateEngine: config.primaryEngine,
candidateReason,
bucket,
};
if (config.primaryEngine === WORKFLOW_ENGINE.NATIVE) {
return { ...selected, reason: candidateReason };
}
return runtime.executionHandoff.enabled
? { ...selected, engine: config.primaryEngine, reason: candidateReason }
: { ...selected, reason: 'execution_gate_disabled', dryRun: true };
}
return base;
},
@@ -362,6 +414,7 @@ export const orchestratorAdminConfigInternals = {
CONFIG_SCOPE,
CONFIG_TABLE,
engineCatalog,
EXECUTION_HANDOFF_IMPLEMENTED,
probeServiceHealth,
runtimeState,
};
+50 -2
View File
@@ -69,8 +69,11 @@ test('orchestrator config persists versioned admin updates and selects canary us
userId: 'user-1',
workflowName: 'code-run-v1',
});
assert.equal(selected.engine, 'langgraph');
assert.equal(selected.reason, 'user_allowlist');
assert.equal(selected.engine, 'native');
assert.equal(selected.candidateEngine, 'langgraph');
assert.equal(selected.reason, 'execution_gate_disabled');
assert.equal(selected.candidateReason, 'user_allowlist');
assert.equal(selected.dryRun, true);
const native = await service.selectEngine({
runId: 'run-2',
@@ -78,9 +81,54 @@ test('orchestrator config persists versioned admin updates and selects canary us
workflowName: 'code-run-v1',
});
assert.equal(native.engine, 'native');
assert.equal(native.candidateEngine, 'native');
assert.equal(native.reason, 'canary_not_selected');
});
test('orchestrator execution handoff remains hard-disabled when Active is requested', async () => {
const service = createOrchestratorAdminConfigService(createPool(), { env: {} });
await service.updateAdminConfig({
mode: 'active',
serviceUrl: 'http://127.0.0.1:8093',
});
const runtime = await service.getRuntimeState();
assert.equal(runtime.runtime.plansLangGraph, true);
assert.equal(runtime.runtime.executesLangGraph, false);
assert.deepEqual(runtime.runtime.executionHandoff, {
implemented: false,
requested: true,
enabled: false,
reason: 'phase3_dry_run_only',
});
const selected = await service.selectEngine({
runId: 'run-1',
userId: 'user-1',
workflowName: 'code-run-v1',
});
assert.equal(selected.engine, 'native');
assert.equal(selected.candidateEngine, 'langgraph');
assert.equal(selected.dryRun, true);
});
test('orchestrator keeps a Native primary as a non-dry-run Native selection', async () => {
const service = createOrchestratorAdminConfigService(createPool(), { env: {} });
await service.updateAdminConfig({
mode: 'active',
primaryEngine: 'native',
});
const selected = await service.selectEngine({
runId: 'run-native',
workflowName: 'code-run-v1',
});
assert.equal(selected.engine, 'native');
assert.equal(selected.candidateEngine, 'native');
assert.equal(selected.reason, 'active');
assert.equal(selected.dryRun, false);
});
test('orchestrator emergency kill switch always forces native selection', async () => {
const service = createOrchestratorAdminConfigService(createPool(), {
env: { MEMIND_ORCHESTRATOR_KILL_SWITCH: '1' },
+143
View File
@@ -0,0 +1,143 @@
import {
WORKFLOW_ENGINE,
normalizeRunSpec,
} from './contracts.mjs';
const EXECUTION_REQUEST_VERSION = 'workflow-execution-request-v1';
const EXECUTION_DECISION_VERSION = 'workflow-execution-decision-v1';
const MAX_IDEMPOTENCY_KEY_CHARACTERS = 200;
function clampTimeoutMs(value, fallback = 30_000) {
const number = Number(value);
if (!Number.isFinite(number)) return fallback;
return Math.min(60_000, Math.max(500, Math.floor(number)));
}
function normalizeIdempotencyKey(value) {
return String(value ?? '').trim().slice(0, MAX_IDEMPOTENCY_KEY_CHARACTERS);
}
export function normalizeWorkflowExecutionRequest(input = {}) {
const source = input && typeof input === 'object' && !Array.isArray(input) ? input : {};
const spec = normalizeRunSpec(source.spec ?? source.runSpec ?? source);
return {
version: EXECUTION_REQUEST_VERSION,
spec,
idempotencyKey: normalizeIdempotencyKey(source.idempotencyKey),
authorization: {
executionAllowed: source.authorization?.executionAllowed === true,
actorId: String(source.authorization?.actorId ?? '').trim() || null,
},
controls: {
timeoutMs: clampTimeoutMs(source.controls?.timeoutMs),
fallbackAllowed: source.controls?.fallbackAllowed !== false,
cancellationAllowed: source.controls?.cancellationAllowed !== false,
},
};
}
function disabledHandoffError(decision) {
const error = new Error(
`Workflow execution handoff is disabled; ${decision.effectiveEngine} remains the executor`,
);
error.code = 'WORKFLOW_EXECUTION_HANDOFF_DISABLED';
error.status = 409;
error.decision = decision;
return error;
}
/**
* Phase 3 boundary adapter.
*
* It turns control-plane routing into a framework-neutral, auditable decision.
* It intentionally has no remote dispatch path yet: Native remains owned by the
* existing Portal Agent Run and non-Native candidates are dry-run only.
*/
export function createWorkflowExecutionAdapter({
configService,
engineRegistry,
} = {}) {
if (!configService?.selectEngine) {
throw new Error('Workflow execution adapter requires orchestrator config service');
}
if (!engineRegistry?.has) {
throw new Error('Workflow execution adapter requires workflow engine registry');
}
async function plan(input) {
const request = normalizeWorkflowExecutionRequest(input);
const selection = await configService.selectEngine({
runId: request.spec.runId,
requestId: request.spec.requestId,
userId: request.spec.subject.userId,
workflowName: request.spec.workflow.name,
});
const candidateEngine = selection.candidateEngine ?? selection.engine ?? WORKFLOW_ENGINE.NATIVE;
const nonNativeCandidate = candidateEngine !== WORKFLOW_ENGINE.NATIVE;
const gates = {
implementation: false,
routingSelected: nonNativeCandidate,
engineRegistered: engineRegistry.has(candidateEngine),
idempotencyKeyPresent: Boolean(request.idempotencyKey),
executionAuthorized: request.authorization.executionAllowed,
killSwitchOpen: selection.reason !== 'kill_switch',
};
return {
version: EXECUTION_DECISION_VERSION,
runId: request.spec.runId,
requestId: request.spec.requestId,
mode: selection.mode,
candidateEngine,
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
fallbackEngine: selection.fallbackEngine ?? WORKFLOW_ENGINE.NATIVE,
dryRun: nonNativeCandidate,
handoffAllowed: false,
reason: nonNativeCandidate
? 'execution_gate_disabled'
: selection.reason ?? 'native_selected',
candidateReason: selection.candidateReason ?? null,
configVersion: selection.configVersion,
gates,
controls: request.controls,
};
}
return {
plan,
async start(input) {
const decision = await plan(input);
if (decision.candidateEngine === WORKFLOW_ENGINE.NATIVE) {
return {
dispatched: false,
reason: 'native_execution_owned_by_portal',
decision,
};
}
throw disabledHandoffError(decision);
},
async resume() {
throw disabledHandoffError({
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
reason: 'execution_gate_disabled',
});
},
async cancel() {
throw disabledHandoffError({
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
reason: 'execution_gate_disabled',
});
},
};
}
export const workflowExecutionAdapterInternals = {
EXECUTION_DECISION_VERSION,
EXECUTION_REQUEST_VERSION,
MAX_IDEMPOTENCY_KEY_CHARACTERS,
clampTimeoutMs,
disabledHandoffError,
normalizeIdempotencyKey,
};
@@ -0,0 +1,113 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createWorkflowEngineRegistry } from './engine-registry.mjs';
import {
createWorkflowExecutionAdapter,
normalizeWorkflowExecutionRequest,
} from './execution-adapter.mjs';
function engine(id) {
return {
id,
start() {
throw new Error('must not dispatch during Phase 3 dry-run');
},
resume() {},
cancel() {},
getState() {},
async *streamEvents() {},
};
}
function executionRequest(overrides = {}) {
return {
spec: {
runId: 'run-1',
requestId: 'request-1',
workflow: { name: 'code-run-v1', version: 1 },
subject: { userId: 'user-1' },
},
idempotencyKey: 'agent-run:run-1:attempt-1',
authorization: { executionAllowed: true, actorId: 'user-1' },
controls: { timeoutMs: 10_000 },
...overrides,
};
}
test('workflow execution request normalizes authorization, limits and idempotency', () => {
const request = normalizeWorkflowExecutionRequest(executionRequest({
idempotencyKey: ` ${'x'.repeat(250)} `,
controls: { timeoutMs: 120_000, fallbackAllowed: false },
}));
assert.equal(request.version, 'workflow-execution-request-v1');
assert.equal(request.idempotencyKey.length, 200);
assert.equal(request.authorization.executionAllowed, true);
assert.equal(request.controls.timeoutMs, 60_000);
assert.equal(request.controls.fallbackAllowed, false);
});
test('workflow execution adapter records a LangGraph candidate but keeps Native effective', async () => {
const registry = createWorkflowEngineRegistry([engine('native'), engine('langgraph')]);
const adapter = createWorkflowExecutionAdapter({
engineRegistry: registry,
configService: {
async selectEngine() {
return {
engine: 'native',
candidateEngine: 'langgraph',
fallbackEngine: 'native',
mode: 'canary',
reason: 'execution_gate_disabled',
candidateReason: 'user_allowlist',
configVersion: 4,
};
},
},
});
const decision = await adapter.plan(executionRequest());
assert.equal(decision.version, 'workflow-execution-decision-v1');
assert.equal(decision.candidateEngine, 'langgraph');
assert.equal(decision.effectiveEngine, 'native');
assert.equal(decision.dryRun, true);
assert.equal(decision.handoffAllowed, false);
assert.equal(decision.candidateReason, 'user_allowlist');
assert.deepEqual(decision.gates, {
implementation: false,
routingSelected: true,
engineRegistered: true,
idempotencyKeyPresent: true,
executionAuthorized: true,
killSwitchOpen: true,
});
await assert.rejects(
() => adapter.start(executionRequest()),
(error) => error.code === 'WORKFLOW_EXECUTION_HANDOFF_DISABLED'
&& error.decision.effectiveEngine === 'native',
);
});
test('workflow execution adapter leaves native dispatch under Portal ownership', async () => {
const registry = createWorkflowEngineRegistry([engine('native')]);
const adapter = createWorkflowExecutionAdapter({
engineRegistry: registry,
configService: {
async selectEngine() {
return {
engine: 'native',
candidateEngine: 'native',
fallbackEngine: 'native',
mode: 'off',
reason: 'mode_off',
configVersion: 1,
};
},
},
});
const result = await adapter.start(executionRequest());
assert.equal(result.dispatched, false);
assert.equal(result.reason, 'native_execution_owned_by_portal');
assert.equal(result.decision.effectiveEngine, 'native');
});
+12
View File
@@ -54,6 +54,18 @@ export function createWorkflowShadowObserver({
observed: false,
reason: selection.reason,
mode: selection.mode,
executionPlan: selection.dryRun ? {
version: 'workflow-execution-decision-v1',
candidateEngine: selection.candidateEngine ?? WORKFLOW_ENGINE.NATIVE,
effectiveEngine: selection.engine ?? WORKFLOW_ENGINE.NATIVE,
fallbackEngine: selection.fallbackEngine ?? WORKFLOW_ENGINE.NATIVE,
reason: selection.reason,
candidateReason: selection.candidateReason ?? null,
configVersion: selection.configVersion ?? null,
bucket: selection.bucket ?? null,
dryRun: true,
handoffAllowed: false,
} : null,
};
}
@@ -35,6 +35,56 @@ test('shadow observer skips without creating a remote client when mode does not
observed: false,
reason: 'mode_off',
mode: 'off',
executionPlan: null,
});
assert.equal(fetchCalls, 0);
});
test('shadow boundary returns an auditable dry-run plan without calling LangGraph', async () => {
let fetchCalls = 0;
const observer = createWorkflowShadowObserver({
configService: {
async selectEngine() {
return {
engine: 'native',
candidateEngine: 'langgraph',
shadowEngine: null,
fallbackEngine: 'native',
reason: 'execution_gate_disabled',
candidateReason: 'user_allowlist',
mode: 'canary',
configVersion: 8,
bucket: 42,
dryRun: true,
};
},
async getRuntimeState() {
throw new Error('must not load runtime config');
},
},
fetchImpl: async () => {
fetchCalls += 1;
return jsonResponse({});
},
});
const result = await observer({
runId: 'run-dry-1',
requestId: 'request-dry-1',
userId: 'user-1',
});
assert.equal(result.observed, false);
assert.deepEqual(result.executionPlan, {
version: 'workflow-execution-decision-v1',
candidateEngine: 'langgraph',
effectiveEngine: 'native',
fallbackEngine: 'native',
reason: 'execution_gate_disabled',
candidateReason: 'user_allowlist',
configVersion: 8,
bucket: 42,
dryRun: true,
handoffAllowed: false,
});
assert.equal(fetchCalls, 0);
});