feat: harden orchestrator execution runtime
This commit is contained in:
john
@@ -7,11 +7,11 @@ import {
|
||||
normalizeWorkflowName,
|
||||
stableRolloutBucket,
|
||||
} from './contracts.mjs';
|
||||
import { listPhase3ExecutorAdapters } from './executor-gateway.mjs';
|
||||
import { listPhase5ExecutorAdapters } from './executor-gateway.mjs';
|
||||
|
||||
const CONFIG_TABLE = 'h5_orchestrator_admin_config';
|
||||
const CONFIG_SCOPE = 'global';
|
||||
const EXECUTION_HANDOFF_IMPLEMENTED = false;
|
||||
const EXECUTION_HANDOFF_IMPLEMENTED = true;
|
||||
|
||||
function normalizeBoolean(value, fallback = false) {
|
||||
if (value == null || value === '') return fallback;
|
||||
@@ -75,6 +75,7 @@ export function defaultOrchestratorConfig() {
|
||||
workflowAllowlist: [...DEFAULT_ORCHESTRATED_WORKFLOWS],
|
||||
fallbackToNative: true,
|
||||
requireHealthy: true,
|
||||
executionEnabled: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -105,11 +106,16 @@ export function normalizeOrchestratorConfig(input = {}, fallback = defaultOrches
|
||||
: [...DEFAULT_ORCHESTRATED_WORKFLOWS],
|
||||
fallbackToNative: normalizeBoolean(source.fallbackToNative, base.fallbackToNative),
|
||||
requireHealthy: normalizeBoolean(source.requireHealthy, base.requireHealthy),
|
||||
executionEnabled: normalizeBoolean(source.executionEnabled, base.executionEnabled),
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeState(config, env = process.env) {
|
||||
const killSwitch = normalizeBoolean(env.MEMIND_ORCHESTRATOR_KILL_SWITCH, false);
|
||||
const environmentExecutionGate = normalizeBoolean(
|
||||
env.MEMIND_ORCHESTRATOR_EXECUTION_HANDOFF_ENABLED,
|
||||
false,
|
||||
);
|
||||
const configured = config.primaryEngine !== WORKFLOW_ENGINE.LANGGRAPH || Boolean(config.serviceUrl);
|
||||
let reason = null;
|
||||
if (killSwitch) reason = 'kill_switch';
|
||||
@@ -117,10 +123,12 @@ function runtimeState(config, env = process.env) {
|
||||
else if (!configured) reason = 'service_url_missing';
|
||||
const executionModeSelected = [ORCHESTRATOR_MODE.CANARY, ORCHESTRATOR_MODE.ACTIVE]
|
||||
.includes(config.mode);
|
||||
const executionHandoffRequested = executionModeSelected
|
||||
const executionHandoffRequested = config.executionEnabled
|
||||
&& executionModeSelected
|
||||
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH;
|
||||
const executionHandoffEnabled = !reason
|
||||
&& EXECUTION_HANDOFF_IMPLEMENTED
|
||||
&& environmentExecutionGate
|
||||
&& executionHandoffRequested
|
||||
&& config.primaryEngine === WORKFLOW_ENGINE.LANGGRAPH;
|
||||
return {
|
||||
@@ -147,9 +155,12 @@ function runtimeState(config, env = process.env) {
|
||||
enabled: executionHandoffEnabled,
|
||||
reason: executionHandoffEnabled
|
||||
? null
|
||||
: EXECUTION_HANDOFF_IMPLEMENTED
|
||||
? 'execution_handoff_not_requested'
|
||||
: 'phase3_dry_run_only',
|
||||
: !EXECUTION_HANDOFF_IMPLEMENTED
|
||||
? 'execution_handoff_not_implemented'
|
||||
: !environmentExecutionGate
|
||||
? 'environment_execution_gate_disabled'
|
||||
: 'execution_handoff_not_requested',
|
||||
environmentGate: environmentExecutionGate,
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -312,7 +323,7 @@ export function createOrchestratorAdminConfigService(pool, {
|
||||
...state,
|
||||
runtime: runtimeState(state.config, env),
|
||||
engines: engineCatalog(state.config),
|
||||
executors: listPhase3ExecutorAdapters(),
|
||||
executors: listPhase5ExecutorAdapters(),
|
||||
};
|
||||
},
|
||||
|
||||
@@ -345,7 +356,7 @@ export function createOrchestratorAdminConfigService(pool, {
|
||||
...state,
|
||||
runtime: runtimeState(state.config, env),
|
||||
engines: engineCatalog(state.config),
|
||||
executors: listPhase3ExecutorAdapters(),
|
||||
executors: listPhase5ExecutorAdapters(),
|
||||
...(probe ? { serviceHealth: await probeServiceHealth(state.config, { fetchImpl }) } : {}),
|
||||
};
|
||||
},
|
||||
|
||||
@@ -41,9 +41,10 @@ test('orchestrator config defaults to a disabled native-safe runtime', async ()
|
||||
);
|
||||
assert.equal(state.executors.every((executor) => executor.enabled === false), true);
|
||||
assert.equal(
|
||||
state.executors.every((executor) => executor.dispatchImplemented === false),
|
||||
state.executors.every((executor) => executor.dispatchImplemented === true),
|
||||
true,
|
||||
);
|
||||
assert.equal(state.config.executionEnabled, false);
|
||||
});
|
||||
|
||||
test('orchestrator config normalizes unsafe values and keeps a workflow allowlist', () => {
|
||||
@@ -94,7 +95,7 @@ test('orchestrator config persists versioned admin updates and selects canary us
|
||||
assert.equal(native.reason, 'canary_not_selected');
|
||||
});
|
||||
|
||||
test('orchestrator execution handoff remains hard-disabled when Active is requested', async () => {
|
||||
test('orchestrator execution handoff remains disabled without both admin and environment gates', async () => {
|
||||
const service = createOrchestratorAdminConfigService(createPool(), { env: {} });
|
||||
await service.updateAdminConfig({
|
||||
mode: 'active',
|
||||
@@ -105,10 +106,11 @@ test('orchestrator execution handoff remains hard-disabled when Active is reques
|
||||
assert.equal(runtime.runtime.plansLangGraph, true);
|
||||
assert.equal(runtime.runtime.executesLangGraph, false);
|
||||
assert.deepEqual(runtime.runtime.executionHandoff, {
|
||||
implemented: false,
|
||||
requested: true,
|
||||
implemented: true,
|
||||
requested: false,
|
||||
enabled: false,
|
||||
reason: 'phase3_dry_run_only',
|
||||
reason: 'environment_execution_gate_disabled',
|
||||
environmentGate: false,
|
||||
});
|
||||
|
||||
const selected = await service.selectEngine({
|
||||
@@ -121,6 +123,30 @@ test('orchestrator execution handoff remains hard-disabled when Active is reques
|
||||
assert.equal(selected.dryRun, true);
|
||||
});
|
||||
|
||||
test('orchestrator selects LangGraph only when admin and environment execution gates agree', async () => {
|
||||
const service = createOrchestratorAdminConfigService(createPool(), {
|
||||
env: { MEMIND_ORCHESTRATOR_EXECUTION_HANDOFF_ENABLED: '1' },
|
||||
});
|
||||
await service.updateAdminConfig({
|
||||
mode: 'canary',
|
||||
serviceUrl: 'http://127.0.0.1:8093',
|
||||
executionEnabled: true,
|
||||
userAllowlist: ['user-1'],
|
||||
});
|
||||
const runtime = await service.getRuntimeState();
|
||||
assert.equal(runtime.runtime.executesLangGraph, true);
|
||||
assert.equal(runtime.runtime.executionHandoff.environmentGate, true);
|
||||
const selected = await service.selectEngine({
|
||||
runId: 'run-1',
|
||||
userId: 'user-1',
|
||||
workflowName: 'code-run-v1',
|
||||
});
|
||||
assert.equal(selected.engine, 'langgraph');
|
||||
assert.equal(selected.candidateEngine, 'langgraph');
|
||||
assert.equal(selected.reason, 'user_allowlist');
|
||||
assert.equal(selected.dryRun, false);
|
||||
});
|
||||
|
||||
test('orchestrator keeps a Native primary as a non-dry-run Native selection', async () => {
|
||||
const service = createOrchestratorAdminConfigService(createPool(), { env: {} });
|
||||
await service.updateAdminConfig({
|
||||
|
||||
@@ -22,9 +22,49 @@ function authorize(serviceToken) {
|
||||
};
|
||||
}
|
||||
|
||||
function authorizeWorker(workerToken) {
|
||||
const expected = String(workerToken ?? '').trim();
|
||||
return (request, response, next) => {
|
||||
if (!expected) return next();
|
||||
if (request.get('x-orchestrator-worker-token') === expected) return next();
|
||||
return response.status(401).json({
|
||||
error: {
|
||||
code: 'WORKER_UNAUTHORIZED',
|
||||
message: 'Missing or invalid worker token',
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function prometheusMetrics(metrics) {
|
||||
const lines = [
|
||||
'# HELP memind_orchestrator_executor_jobs Executor jobs by state.',
|
||||
'# TYPE memind_orchestrator_executor_jobs gauge',
|
||||
];
|
||||
for (const [status, count] of Object.entries(metrics.queue?.counts ?? {})) {
|
||||
lines.push(`memind_orchestrator_executor_jobs{status="${status}"} ${Number(count) || 0}`);
|
||||
}
|
||||
lines.push(
|
||||
'# HELP memind_orchestrator_executor_claimable Claimable executor jobs.',
|
||||
'# TYPE memind_orchestrator_executor_claimable gauge',
|
||||
`memind_orchestrator_executor_claimable ${Number(metrics.queue?.claimable) || 0}`,
|
||||
'# HELP memind_orchestrator_executor_expired_leases Expired executor leases.',
|
||||
'# TYPE memind_orchestrator_executor_expired_leases gauge',
|
||||
`memind_orchestrator_executor_expired_leases ${Number(metrics.queue?.expiredLeases) || 0}`,
|
||||
'# HELP memind_orchestrator_executor_workers Healthy registered workers.',
|
||||
'# TYPE memind_orchestrator_executor_workers gauge',
|
||||
`memind_orchestrator_executor_workers ${Number(metrics.queue?.healthyWorkers) || 0}`,
|
||||
'# HELP memind_orchestrator_execution_enabled Whether executor queue admission is enabled.',
|
||||
'# TYPE memind_orchestrator_execution_enabled gauge',
|
||||
`memind_orchestrator_execution_enabled ${metrics.gateway?.executionEnabled ? 1 : 0}`,
|
||||
);
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
export function createOrchestratorApp({
|
||||
runtime,
|
||||
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
||||
workerToken = process.env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
|
||||
} = {}) {
|
||||
if (!runtime) throw new Error('Orchestrator app requires runtime');
|
||||
const app = express();
|
||||
@@ -42,6 +82,16 @@ export function createOrchestratorApp({
|
||||
});
|
||||
}
|
||||
});
|
||||
app.get('/metrics', async (_request, response, next) => {
|
||||
try {
|
||||
const metrics = await runtime.metrics();
|
||||
return response
|
||||
.type('text/plain; version=0.0.4; charset=utf-8')
|
||||
.send(prometheusMetrics(metrics));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.use('/v1', authorize(serviceToken));
|
||||
|
||||
@@ -53,6 +103,15 @@ export function createOrchestratorApp({
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/v1/executor-jobs', async (request, response, next) => {
|
||||
try {
|
||||
const result = await runtime.submitExecutorJob(request.body);
|
||||
return response.status(result.created ? 202 : 200).json(result);
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/v1/runs/:runId', async (request, response, next) => {
|
||||
try {
|
||||
const state = await runtime.getState(request.params.runId);
|
||||
@@ -122,6 +181,105 @@ export function createOrchestratorApp({
|
||||
}
|
||||
});
|
||||
|
||||
const requireWorker = authorizeWorker(workerToken);
|
||||
|
||||
app.post('/v1/workers/claim', requireWorker, async (request, response, next) => {
|
||||
try {
|
||||
const claim = await runtime.claimExecutorJob(request.body);
|
||||
return claim ? response.json(claim) : response.status(204).end();
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post(
|
||||
'/v1/workers/jobs/:jobId/start',
|
||||
requireWorker,
|
||||
async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.startExecutorJob(request.params.jobId, request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.post(
|
||||
'/v1/workers/jobs/:jobId/heartbeat',
|
||||
requireWorker,
|
||||
async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.heartbeatExecutorJob(request.params.jobId, request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.post(
|
||||
'/v1/workers/jobs/:jobId/progress',
|
||||
requireWorker,
|
||||
async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.progressExecutorJob(request.params.jobId, request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.post(
|
||||
'/v1/workers/jobs/:jobId/complete',
|
||||
requireWorker,
|
||||
async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.completeExecutorJob(request.params.jobId, request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.post(
|
||||
'/v1/workers/jobs/:jobId/fail',
|
||||
requireWorker,
|
||||
async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.failExecutorJob(request.params.jobId, request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
app.post('/v1/workers/recover-expired', requireWorker, async (request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.recoverExpiredExecutorJobs(request.body));
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.get('/v1/workers/stats', requireWorker, async (_request, response, next) => {
|
||||
try {
|
||||
return response.json(await runtime.getExecutorQueueStats());
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.post('/v1/executor-jobs/:jobId/cancel', async (request, response, next) => {
|
||||
try {
|
||||
const job = await runtime.cancelExecutorJob(request.params.jobId, request.body);
|
||||
if (!job) {
|
||||
return response.status(404).json({ error: { code: 'EXECUTOR_JOB_NOT_FOUND' } });
|
||||
}
|
||||
return response.json(job);
|
||||
} catch (error) {
|
||||
return next(error);
|
||||
}
|
||||
});
|
||||
|
||||
app.use((error, _request, response, _next) => {
|
||||
response.status(errorStatus(error)).json({
|
||||
error: {
|
||||
@@ -136,5 +294,7 @@ export function createOrchestratorApp({
|
||||
|
||||
export const orchestratorAppInternals = {
|
||||
authorize,
|
||||
authorizeWorker,
|
||||
errorStatus,
|
||||
prometheusMetrics,
|
||||
};
|
||||
|
||||
@@ -125,3 +125,96 @@ test('orchestrator HTTP API reports missing runs and observe-only violations', a
|
||||
assert.equal(rejected.status, 422);
|
||||
assert.equal((await rejected.json()).error.code, 'WORKFLOW_OBSERVE_ONLY_REQUIRED');
|
||||
});
|
||||
|
||||
test('executor worker HTTP protocol requires both service and worker credentials', async (t) => {
|
||||
const runtime = createLangGraphOrchestratorRuntime({
|
||||
checkpointer: new MemorySaver(),
|
||||
executionEnabled: true,
|
||||
enabledExecutors: ['goosed'],
|
||||
});
|
||||
const server = await listen(createOrchestratorApp({
|
||||
runtime,
|
||||
serviceToken: 'service-token',
|
||||
workerToken: 'worker-token',
|
||||
}));
|
||||
t.after(server.close);
|
||||
const serviceHeaders = {
|
||||
authorization: 'Bearer service-token',
|
||||
'content-type': 'application/json',
|
||||
};
|
||||
const submitted = await fetch(`${server.baseUrl}/v1/executor-jobs`, {
|
||||
method: 'POST',
|
||||
headers: serviceHeaders,
|
||||
body: JSON.stringify({
|
||||
jobId: 'http-worker-job-1',
|
||||
idempotencyKey: 'http-worker-idem-1',
|
||||
executor: 'goosed',
|
||||
task: {
|
||||
instruction: 'Run the canary',
|
||||
workspaceRef: { kind: 'workspace-alias', id: 'canary' },
|
||||
},
|
||||
subject: { tenantId: 'tenant-1', userId: 'user-1' },
|
||||
authorization: { executionAllowed: true },
|
||||
policy: { sideEffectsAllowed: true },
|
||||
}),
|
||||
});
|
||||
assert.equal(submitted.status, 202);
|
||||
assert.equal((await submitted.json()).job.status, 'queued');
|
||||
|
||||
const unauthorizedWorker = await fetch(`${server.baseUrl}/v1/workers/claim`, {
|
||||
method: 'POST',
|
||||
headers: serviceHeaders,
|
||||
body: JSON.stringify({ workerId: 'worker-1', executors: ['goosed'] }),
|
||||
});
|
||||
assert.equal(unauthorizedWorker.status, 401);
|
||||
assert.equal((await unauthorizedWorker.json()).error.code, 'WORKER_UNAUTHORIZED');
|
||||
|
||||
const workerHeaders = {
|
||||
...serviceHeaders,
|
||||
'x-orchestrator-worker-token': 'worker-token',
|
||||
};
|
||||
const claimResponse = await fetch(`${server.baseUrl}/v1/workers/claim`, {
|
||||
method: 'POST',
|
||||
headers: workerHeaders,
|
||||
body: JSON.stringify({ workerId: 'worker-1', executors: ['goosed'] }),
|
||||
});
|
||||
assert.equal(claimResponse.status, 200);
|
||||
const claim = await claimResponse.json();
|
||||
assert.equal(claim.job.request.task.instruction, 'Run the canary');
|
||||
|
||||
const started = await fetch(
|
||||
`${server.baseUrl}/v1/workers/jobs/http-worker-job-1/start`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: workerHeaders,
|
||||
body: JSON.stringify({ leaseToken: claim.leaseToken }),
|
||||
},
|
||||
);
|
||||
assert.equal(started.status, 200);
|
||||
|
||||
const completed = await fetch(
|
||||
`${server.baseUrl}/v1/workers/jobs/http-worker-job-1/complete`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: workerHeaders,
|
||||
body: JSON.stringify({
|
||||
leaseToken: claim.leaseToken,
|
||||
result: {
|
||||
summary: 'completed',
|
||||
artifactRefs: [{ kind: 'canary-result', id: 'result-1' }],
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(completed.status, 200);
|
||||
assert.equal((await completed.json()).status, 'succeeded');
|
||||
|
||||
const projected = await fetch(
|
||||
`${server.baseUrl}/v1/executor-jobs/http-worker-job-1`,
|
||||
{ headers: { authorization: 'Bearer service-token' } },
|
||||
);
|
||||
const projectedBody = await projected.json();
|
||||
assert.equal(projectedBody.status, 'succeeded');
|
||||
assert.equal('request' in projectedBody, false);
|
||||
assert.equal('token' in (projectedBody.lease ?? {}), false);
|
||||
});
|
||||
|
||||
@@ -61,15 +61,24 @@ function eventFor(state, type, data = null) {
|
||||
});
|
||||
}
|
||||
|
||||
function validateNode(state) {
|
||||
function validateNode(state, { executionEnabled = false } = {}) {
|
||||
const spec = normalizeRunSpec(state.spec);
|
||||
if (spec.workflow.name !== CODE_RUN_SHADOW_WORKFLOW) {
|
||||
const error = new Error(`Unsupported workflow: ${spec.workflow.name}`);
|
||||
error.code = 'WORKFLOW_NOT_SUPPORTED';
|
||||
throw error;
|
||||
}
|
||||
if (spec.policy.executionMode !== 'observe-only' || spec.policy.sideEffectsAllowed !== false) {
|
||||
const error = new Error('Phase 2 LangGraph service accepts observe-only runs');
|
||||
const observeOnly = spec.policy.executionMode === 'observe-only'
|
||||
&& spec.policy.sideEffectsAllowed === false;
|
||||
const active = spec.policy.executionMode === 'active'
|
||||
&& spec.policy.sideEffectsAllowed === true
|
||||
&& executionEnabled;
|
||||
if (!observeOnly && !active) {
|
||||
const error = new Error(
|
||||
executionEnabled
|
||||
? 'LangGraph run policy must be observe-only or explicitly active'
|
||||
: 'LangGraph execution is disabled; only observe-only runs are accepted',
|
||||
);
|
||||
error.code = 'WORKFLOW_OBSERVE_ONLY_REQUIRED';
|
||||
throw error;
|
||||
}
|
||||
@@ -79,7 +88,7 @@ function validateNode(state) {
|
||||
phase: 'validated',
|
||||
events: [eventFor(state, 'workflow_validated', {
|
||||
workflow: spec.workflow,
|
||||
executionMode: 'observe-only',
|
||||
executionMode: spec.policy.executionMode,
|
||||
})],
|
||||
};
|
||||
}
|
||||
@@ -88,10 +97,12 @@ async function planNode(state, executorGateway) {
|
||||
const taskType = String(state.spec.input?.taskType ?? '').trim() || 'code-change';
|
||||
const instruction = String(state.spec.input?.instruction ?? '');
|
||||
const executorIdentity = executorIdentityForRun(state.spec.runId);
|
||||
const active = state.spec.policy.executionMode === 'active';
|
||||
const executor = String(state.spec.input?.executor ?? 'goosed').trim().toLowerCase();
|
||||
const executorJob = await executorGateway.createJob({
|
||||
jobId: executorIdentity.jobId,
|
||||
idempotencyKey: executorIdentity.idempotencyKey,
|
||||
executor: 'goosed',
|
||||
executor,
|
||||
task: {
|
||||
type: taskType,
|
||||
instruction,
|
||||
@@ -100,17 +111,18 @@ async function planNode(state, executorGateway) {
|
||||
},
|
||||
subject: state.spec.subject,
|
||||
authorization: {
|
||||
executionAllowed: false,
|
||||
executionAllowed: active,
|
||||
actorId: state.spec.subject?.userId ?? null,
|
||||
},
|
||||
policy: {
|
||||
sideEffectsAllowed: false,
|
||||
networkAllowed: false,
|
||||
sideEffectsAllowed: active && state.spec.policy.sideEffectsAllowed === true,
|
||||
networkAllowed: active && state.spec.policy.networkAllowed === true,
|
||||
},
|
||||
controls: {
|
||||
timeoutMs: state.spec.limits?.timeoutMs,
|
||||
cancellationAllowed: true,
|
||||
fallbackExecutor: 'aider',
|
||||
maxAttempts: state.spec.limits?.maxAttempts,
|
||||
},
|
||||
metadata: {
|
||||
source: 'langgraph-shadow-plan',
|
||||
@@ -119,7 +131,7 @@ async function planNode(state, executorGateway) {
|
||||
});
|
||||
const plan = {
|
||||
taskType,
|
||||
executorAdapter: 'native-agent-run',
|
||||
executorAdapter: active ? 'executor-gateway' : 'native-agent-run',
|
||||
executorJob: {
|
||||
kind: 'executor-job',
|
||||
id: executorJob.job.jobId,
|
||||
@@ -148,27 +160,36 @@ async function planNode(state, executorGateway) {
|
||||
}
|
||||
|
||||
function finalizeNode(state) {
|
||||
const queued = state.plan.executorJob.status === 'queued';
|
||||
const result = {
|
||||
observed: true,
|
||||
observed: !queued,
|
||||
executed: false,
|
||||
executorAdapter: state.plan.executorAdapter,
|
||||
taskType: state.plan.taskType,
|
||||
};
|
||||
return {
|
||||
status: 'succeeded',
|
||||
phase: 'completed',
|
||||
status: queued ? 'waiting' : 'succeeded',
|
||||
phase: queued ? 'executor_queued' : 'completed',
|
||||
result,
|
||||
events: [eventFor(state, 'workflow_completed', result)],
|
||||
events: [eventFor(
|
||||
state,
|
||||
queued ? 'workflow_executor_queued' : 'workflow_completed',
|
||||
result,
|
||||
)],
|
||||
};
|
||||
}
|
||||
|
||||
export function createCodeRunShadowGraph({ checkpointer, executorGateway } = {}) {
|
||||
export function createCodeRunShadowGraph({
|
||||
checkpointer,
|
||||
executorGateway,
|
||||
executionEnabled = false,
|
||||
} = {}) {
|
||||
if (!checkpointer) throw new Error('Code run shadow graph requires a checkpointer');
|
||||
if (!executorGateway?.createJob) {
|
||||
throw new Error('Code run shadow graph requires an Executor Gateway');
|
||||
}
|
||||
return new StateGraph(ShadowState)
|
||||
.addNode('validate_run', validateNode)
|
||||
.addNode('validate_run', (state) => validateNode(state, { executionEnabled }))
|
||||
.addNode('build_plan', (state) => planNode(state, executorGateway))
|
||||
.addNode('finalize_run', finalizeNode)
|
||||
.addEdge(START, 'validate_run')
|
||||
|
||||
@@ -47,11 +47,8 @@ function disabledHandoffError(decision) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Phase 3 boundary adapter.
|
||||
*
|
||||
* It turns control-plane routing into a framework-neutral, auditable decision.
|
||||
* It intentionally has no remote dispatch path yet: Native remains owned by the
|
||||
* existing Portal Agent Run and non-Native candidates are dry-run only.
|
||||
* Framework-neutral control-plane adapter. Routing, authorization and the
|
||||
* environment/admin gates are evaluated before a remote engine can be invoked.
|
||||
*/
|
||||
export function createWorkflowExecutionAdapter({
|
||||
configService,
|
||||
@@ -74,27 +71,32 @@ export function createWorkflowExecutionAdapter({
|
||||
});
|
||||
const candidateEngine = selection.candidateEngine ?? selection.engine ?? WORKFLOW_ENGINE.NATIVE;
|
||||
const nonNativeCandidate = candidateEngine !== WORKFLOW_ENGINE.NATIVE;
|
||||
const selectedEngine = selection.engine ?? WORKFLOW_ENGINE.NATIVE;
|
||||
const nonNativeSelected = selectedEngine !== WORKFLOW_ENGINE.NATIVE;
|
||||
const gates = {
|
||||
implementation: false,
|
||||
routingSelected: nonNativeCandidate,
|
||||
engineRegistered: engineRegistry.has(candidateEngine),
|
||||
implementation: true,
|
||||
routingSelected: nonNativeSelected,
|
||||
engineRegistered: engineRegistry.has(selectedEngine),
|
||||
idempotencyKeyPresent: Boolean(request.idempotencyKey),
|
||||
executionAuthorized: request.authorization.executionAllowed,
|
||||
killSwitchOpen: selection.reason !== 'kill_switch',
|
||||
};
|
||||
const handoffAllowed = nonNativeSelected && Object.values(gates).every(Boolean);
|
||||
return {
|
||||
version: EXECUTION_DECISION_VERSION,
|
||||
runId: request.spec.runId,
|
||||
requestId: request.spec.requestId,
|
||||
mode: selection.mode,
|
||||
candidateEngine,
|
||||
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
|
||||
effectiveEngine: handoffAllowed ? selectedEngine : WORKFLOW_ENGINE.NATIVE,
|
||||
fallbackEngine: selection.fallbackEngine ?? WORKFLOW_ENGINE.NATIVE,
|
||||
dryRun: nonNativeCandidate,
|
||||
handoffAllowed: false,
|
||||
reason: nonNativeCandidate
|
||||
? 'execution_gate_disabled'
|
||||
: selection.reason ?? 'native_selected',
|
||||
dryRun: nonNativeCandidate && !handoffAllowed,
|
||||
handoffAllowed,
|
||||
reason: handoffAllowed
|
||||
? selection.reason ?? 'execution_handoff_selected'
|
||||
: nonNativeCandidate
|
||||
? selection.reason ?? 'execution_gate_disabled'
|
||||
: selection.reason ?? 'native_selected',
|
||||
candidateReason: selection.candidateReason ?? null,
|
||||
configVersion: selection.configVersion,
|
||||
gates,
|
||||
@@ -106,29 +108,55 @@ export function createWorkflowExecutionAdapter({
|
||||
plan,
|
||||
|
||||
async start(input) {
|
||||
const request = normalizeWorkflowExecutionRequest(input);
|
||||
const decision = await plan(input);
|
||||
if (decision.candidateEngine === WORKFLOW_ENGINE.NATIVE) {
|
||||
if (!decision.handoffAllowed) {
|
||||
if (decision.candidateEngine !== WORKFLOW_ENGINE.NATIVE) {
|
||||
throw disabledHandoffError(decision);
|
||||
}
|
||||
return {
|
||||
dispatched: false,
|
||||
reason: 'native_execution_owned_by_portal',
|
||||
decision,
|
||||
};
|
||||
}
|
||||
throw disabledHandoffError(decision);
|
||||
const engine = engineRegistry.get(decision.effectiveEngine);
|
||||
try {
|
||||
const state = await engine.start(request.spec);
|
||||
return {
|
||||
dispatched: true,
|
||||
engine: decision.effectiveEngine,
|
||||
state,
|
||||
decision,
|
||||
};
|
||||
} catch (error) {
|
||||
error.code = error.code ?? 'WORKFLOW_EXECUTION_HANDOFF_FAILED';
|
||||
error.decision = decision;
|
||||
error.fallbackRequested = request.controls.fallbackAllowed;
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
|
||||
async resume() {
|
||||
throw disabledHandoffError({
|
||||
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
|
||||
reason: 'execution_gate_disabled',
|
||||
});
|
||||
async resume({ engineId = WORKFLOW_ENGINE.LANGGRAPH, runId, input = null } = {}) {
|
||||
const engine = engineRegistry.get(engineId);
|
||||
if (!engine || engineId === WORKFLOW_ENGINE.NATIVE) {
|
||||
throw disabledHandoffError({
|
||||
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
|
||||
reason: 'execution_gate_disabled',
|
||||
});
|
||||
}
|
||||
return engine.resume(runId, input);
|
||||
},
|
||||
|
||||
async cancel() {
|
||||
throw disabledHandoffError({
|
||||
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
|
||||
reason: 'execution_gate_disabled',
|
||||
});
|
||||
async cancel({ engineId = WORKFLOW_ENGINE.LANGGRAPH, runId, reason = null } = {}) {
|
||||
const engine = engineRegistry.get(engineId);
|
||||
if (!engine || engineId === WORKFLOW_ENGINE.NATIVE) {
|
||||
throw disabledHandoffError({
|
||||
effectiveEngine: WORKFLOW_ENGINE.NATIVE,
|
||||
reason: 'execution_gate_disabled',
|
||||
});
|
||||
}
|
||||
return engine.cancel(runId, reason);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -73,8 +73,8 @@ test('workflow execution adapter records a LangGraph candidate but keeps Native
|
||||
assert.equal(decision.handoffAllowed, false);
|
||||
assert.equal(decision.candidateReason, 'user_allowlist');
|
||||
assert.deepEqual(decision.gates, {
|
||||
implementation: false,
|
||||
routingSelected: true,
|
||||
implementation: true,
|
||||
routingSelected: false,
|
||||
engineRegistered: true,
|
||||
idempotencyKeyPresent: true,
|
||||
executionAuthorized: true,
|
||||
@@ -88,6 +88,42 @@ test('workflow execution adapter records a LangGraph candidate but keeps Native
|
||||
);
|
||||
});
|
||||
|
||||
test('workflow execution adapter dispatches a selected and authorized LangGraph engine', async () => {
|
||||
let started = null;
|
||||
const registry = createWorkflowEngineRegistry([
|
||||
engine('native'),
|
||||
{
|
||||
...engine('langgraph'),
|
||||
start(spec) {
|
||||
started = spec;
|
||||
return { runId: spec.runId, status: 'waiting' };
|
||||
},
|
||||
},
|
||||
]);
|
||||
const adapter = createWorkflowExecutionAdapter({
|
||||
engineRegistry: registry,
|
||||
configService: {
|
||||
async selectEngine() {
|
||||
return {
|
||||
engine: 'langgraph',
|
||||
candidateEngine: 'langgraph',
|
||||
fallbackEngine: 'native',
|
||||
mode: 'canary',
|
||||
reason: 'user_allowlist',
|
||||
candidateReason: 'user_allowlist',
|
||||
configVersion: 5,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
const result = await adapter.start(executionRequest());
|
||||
assert.equal(result.dispatched, true);
|
||||
assert.equal(result.engine, 'langgraph');
|
||||
assert.equal(result.decision.handoffAllowed, true);
|
||||
assert.equal(result.decision.dryRun, false);
|
||||
assert.equal(started.runId, 'run-1');
|
||||
});
|
||||
|
||||
test('workflow execution adapter leaves native dispatch under Portal ownership', async () => {
|
||||
const registry = createWorkflowEngineRegistry([engine('native')]);
|
||||
const adapter = createWorkflowExecutionAdapter({
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
import { spawn as nodeSpawn } from 'node:child_process';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { EventEmitter } from 'node:events';
|
||||
|
||||
const DEFAULT_OUTPUT_LIMIT = 64 * 1024;
|
||||
const DEFAULT_ALLOWED_ENV = Object.freeze(['PATH', 'LANG', 'LC_ALL', 'TMPDIR']);
|
||||
|
||||
function adapterError(code, message, { retryable = false, cause = null } = {}) {
|
||||
const error = new Error(message, cause ? { cause } : undefined);
|
||||
error.code = code;
|
||||
error.retryable = retryable;
|
||||
return error;
|
||||
}
|
||||
|
||||
function appendBounded(current, chunk, limit) {
|
||||
const next = `${current}${Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk ?? '')}`;
|
||||
return next.length <= limit ? next : next.slice(next.length - limit);
|
||||
}
|
||||
|
||||
function isLoopbackHostname(hostname) {
|
||||
const normalized = String(hostname ?? '').trim().toLowerCase();
|
||||
return normalized === 'localhost'
|
||||
|| normalized === '127.0.0.1'
|
||||
|| normalized === '::1'
|
||||
|| normalized === '[::1]'
|
||||
|| normalized.endsWith('.localhost');
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(value, { allowRemote = false } = {}) {
|
||||
const url = new URL(String(value ?? '').trim());
|
||||
if (!['http:', 'https:'].includes(url.protocol)) {
|
||||
throw new Error('Executor adapter URL must use HTTP or HTTPS');
|
||||
}
|
||||
if (!allowRemote && !isLoopbackHostname(url.hostname)) {
|
||||
throw new Error('Remote executor targets require explicit allowRemote');
|
||||
}
|
||||
url.username = '';
|
||||
url.password = '';
|
||||
url.hash = '';
|
||||
return url.toString().replace(/\/$/, '');
|
||||
}
|
||||
|
||||
async function responseError(response, code, prefix) {
|
||||
const text = await response.text().catch(() => '');
|
||||
return adapterError(
|
||||
code,
|
||||
`${prefix} (${response.status})${text ? `: ${text.slice(0, 512)}` : ''}`,
|
||||
{ retryable: response.status >= 500 || response.status === 429 },
|
||||
);
|
||||
}
|
||||
|
||||
async function* parseSse(body) {
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = '';
|
||||
for await (const chunk of body) {
|
||||
buffer += decoder.decode(chunk, { stream: true });
|
||||
while (true) {
|
||||
const boundary = buffer.search(/\r?\n\r?\n/);
|
||||
if (boundary < 0) break;
|
||||
const raw = buffer.slice(0, boundary);
|
||||
const separatorLength = buffer.startsWith('\r\n\r\n', boundary) ? 4 : 2;
|
||||
buffer = buffer.slice(boundary + separatorLength);
|
||||
let event = 'message';
|
||||
const data = [];
|
||||
for (const line of raw.split(/\r?\n/)) {
|
||||
if (line.startsWith('event:')) event = line.slice(6).trim() || 'message';
|
||||
if (line.startsWith('data:')) data.push(line.slice(5).trimStart());
|
||||
}
|
||||
if (data.length) {
|
||||
const text = data.join('\n');
|
||||
let parsed = text;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
// Preserve non-JSON Goosed events as bounded text.
|
||||
}
|
||||
yield { event, data: parsed };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function terminalGoosedEvent(event) {
|
||||
const type = String(
|
||||
event?.event
|
||||
?? event?.data?.type
|
||||
?? event?.data?.event_type
|
||||
?? '',
|
||||
).trim().toLowerCase();
|
||||
return ['finish', 'finished', 'completed', 'error', 'failed'].includes(type);
|
||||
}
|
||||
|
||||
export function createGoosedExecutorAdapter({
|
||||
baseUrl,
|
||||
secret = '',
|
||||
fetchImpl = globalThis.fetch,
|
||||
allowRemote = false,
|
||||
workspaceResolver = async (reference) => reference?.id ?? null,
|
||||
} = {}) {
|
||||
const target = normalizeBaseUrl(baseUrl, { allowRemote });
|
||||
if (typeof fetchImpl !== 'function') throw new Error('Goosed adapter requires fetch');
|
||||
const active = new Map();
|
||||
|
||||
function headers(extra = {}) {
|
||||
return {
|
||||
accept: 'application/json',
|
||||
'content-type': 'application/json',
|
||||
...(secret ? { authorization: `Bearer ${secret}` } : {}),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
async function submit(request, {
|
||||
jobId,
|
||||
signal = null,
|
||||
emit = async () => {},
|
||||
} = {}) {
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort(signal?.reason);
|
||||
if (signal?.aborted) abort();
|
||||
else signal?.addEventListener?.('abort', abort, { once: true });
|
||||
active.set(jobId, controller);
|
||||
try {
|
||||
const workingDir = await workspaceResolver(request.task.workspaceRef);
|
||||
if (!workingDir) {
|
||||
throw adapterError(
|
||||
'GOOSED_WORKSPACE_UNRESOLVED',
|
||||
'Goosed workspace reference could not be resolved',
|
||||
);
|
||||
}
|
||||
const start = await fetchImpl(`${target}/agent/start`, {
|
||||
method: 'POST',
|
||||
headers: headers(),
|
||||
body: JSON.stringify({
|
||||
working_dir: workingDir,
|
||||
enable_context_memory: false,
|
||||
}),
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!start.ok) throw await responseError(start, 'GOOSED_START_FAILED', 'Goosed start failed');
|
||||
const session = await start.json();
|
||||
const sessionId = String(session?.id ?? '').trim();
|
||||
if (!sessionId) {
|
||||
throw adapterError('GOOSED_SESSION_ID_MISSING', 'Goosed start response missing session id');
|
||||
}
|
||||
await emit('executor_job_session_created', {
|
||||
sessionRef: { kind: 'goosed-session', id: sessionId },
|
||||
});
|
||||
|
||||
const eventsResponse = await fetchImpl(
|
||||
`${target}/sessions/${encodeURIComponent(sessionId)}/events`,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: headers({ accept: 'text/event-stream' }),
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
if (!eventsResponse.ok || !eventsResponse.body) {
|
||||
throw await responseError(
|
||||
eventsResponse,
|
||||
'GOOSED_EVENTS_FAILED',
|
||||
'Goosed event stream failed',
|
||||
);
|
||||
}
|
||||
|
||||
const finishPromise = (async () => {
|
||||
let eventCount = 0;
|
||||
let finalEvent = null;
|
||||
for await (const event of parseSse(eventsResponse.body)) {
|
||||
eventCount += 1;
|
||||
const eventType = String(
|
||||
event?.event ?? event?.data?.type ?? event?.data?.event_type ?? 'message',
|
||||
).slice(0, 96);
|
||||
await emit('executor_job_adapter_event', {
|
||||
adapter: 'goosed',
|
||||
eventType,
|
||||
eventCount,
|
||||
});
|
||||
if (terminalGoosedEvent(event)) {
|
||||
finalEvent = event;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { eventCount, finalEvent };
|
||||
})();
|
||||
|
||||
const reply = await fetchImpl(
|
||||
`${target}/sessions/${encodeURIComponent(sessionId)}/reply`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: headers(),
|
||||
body: JSON.stringify({
|
||||
request_id: jobId,
|
||||
user_message: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: request.task.instruction }],
|
||||
},
|
||||
}),
|
||||
signal: controller.signal,
|
||||
},
|
||||
);
|
||||
if (!reply.ok) throw await responseError(reply, 'GOOSED_REPLY_FAILED', 'Goosed reply failed');
|
||||
reply.body?.cancel?.().catch?.(() => {});
|
||||
|
||||
const finished = await finishPromise;
|
||||
const finalType = String(
|
||||
finished.finalEvent?.event
|
||||
?? finished.finalEvent?.data?.type
|
||||
?? finished.finalEvent?.data?.event_type
|
||||
?? '',
|
||||
).toLowerCase();
|
||||
if (['error', 'failed'].includes(finalType)) {
|
||||
throw adapterError('GOOSED_RUN_FAILED', 'Goosed reported a failed terminal event');
|
||||
}
|
||||
return {
|
||||
outcome: 'completed',
|
||||
summary: `Goosed session completed after ${finished.eventCount} events`,
|
||||
artifactRefs: [{ kind: 'goosed-session', id: sessionId }],
|
||||
metrics: { eventCount: finished.eventCount },
|
||||
};
|
||||
} catch (error) {
|
||||
if (controller.signal.aborted || error?.name === 'AbortError') {
|
||||
throw adapterError('GOOSED_CANCELLED', 'Goosed execution was cancelled');
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
signal?.removeEventListener?.('abort', abort);
|
||||
active.delete(jobId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id: 'goosed',
|
||||
label: 'Goosed',
|
||||
kind: 'remote',
|
||||
enabled: true,
|
||||
dispatchImplemented: true,
|
||||
capabilities: ['session-execution', 'streaming', 'cancel'],
|
||||
submit,
|
||||
async cancel(jobId) {
|
||||
const controller = active.get(jobId);
|
||||
if (!controller) return false;
|
||||
controller.abort();
|
||||
return true;
|
||||
},
|
||||
async getState(jobId) {
|
||||
return { active: active.has(jobId) };
|
||||
},
|
||||
async *streamEvents() {
|
||||
throw adapterError(
|
||||
'GOOSED_STREAM_PUSH_ONLY',
|
||||
'Goosed events are delivered through submit emit callbacks',
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveWorkspace(reference, {
|
||||
allowedRoot,
|
||||
aliases = {},
|
||||
} = {}) {
|
||||
const alias = String(reference?.id ?? '').trim();
|
||||
const configured = aliases[alias];
|
||||
if (!configured) {
|
||||
throw adapterError(
|
||||
'EXECUTOR_WORKSPACE_ALIAS_UNKNOWN',
|
||||
`Workspace alias is not configured: ${alias || '<empty>'}`,
|
||||
);
|
||||
}
|
||||
const root = await fs.realpath(allowedRoot);
|
||||
const resolved = await fs.realpath(path.resolve(configured));
|
||||
if (resolved !== root && !resolved.startsWith(`${root}${path.sep}`)) {
|
||||
throw adapterError(
|
||||
'EXECUTOR_WORKSPACE_OUTSIDE_ROOT',
|
||||
'Workspace resolves outside the configured worker root',
|
||||
);
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function safeCommand(value) {
|
||||
const command = String(value ?? '').trim();
|
||||
if (!command || /[\r\n\0]/.test(command)) {
|
||||
throw new Error('Process executor requires a safe command');
|
||||
}
|
||||
return command;
|
||||
}
|
||||
|
||||
export function createProcessExecutorAdapter({
|
||||
id,
|
||||
label = id,
|
||||
command,
|
||||
buildArgs,
|
||||
allowedRoot,
|
||||
workspaceAliases = {},
|
||||
spawnImpl = nodeSpawn,
|
||||
outputLimit = DEFAULT_OUTPUT_LIMIT,
|
||||
allowedEnv = DEFAULT_ALLOWED_ENV,
|
||||
extraEnv = {},
|
||||
artifactCollector = async ({ request }) => [request.task.workspaceRef],
|
||||
} = {}) {
|
||||
const executorId = String(id ?? '').trim().toLowerCase();
|
||||
if (!['aider', 'openhands'].includes(executorId)) {
|
||||
throw new Error('Process executor id must be aider or openhands');
|
||||
}
|
||||
const executable = safeCommand(command);
|
||||
if (!allowedRoot) throw new Error('Process executor requires allowedRoot');
|
||||
if (typeof buildArgs !== 'function') throw new Error('Process executor requires buildArgs');
|
||||
const active = new Map();
|
||||
|
||||
async function submit(request, {
|
||||
jobId,
|
||||
signal = null,
|
||||
emit = async () => {},
|
||||
} = {}) {
|
||||
const cwd = await resolveWorkspace(request.task.workspaceRef, {
|
||||
allowedRoot,
|
||||
aliases: workspaceAliases,
|
||||
});
|
||||
const args = buildArgs(request.task.instruction, request);
|
||||
if (!Array.isArray(args) || args.some((arg) => typeof arg !== 'string')) {
|
||||
throw new Error(`${executorId} buildArgs must return a string array`);
|
||||
}
|
||||
const env = {};
|
||||
for (const key of allowedEnv) {
|
||||
if (process.env[key] != null) env[key] = process.env[key];
|
||||
}
|
||||
Object.assign(env, extraEnv);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let settled = false;
|
||||
const child = spawnImpl(executable, args, {
|
||||
cwd,
|
||||
env,
|
||||
shell: false,
|
||||
detached: false,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
active.set(jobId, child);
|
||||
const cleanup = () => {
|
||||
active.delete(jobId);
|
||||
signal?.removeEventListener?.('abort', onAbort);
|
||||
};
|
||||
const fail = (error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
reject(error);
|
||||
};
|
||||
const onAbort = () => {
|
||||
child.kill('SIGTERM');
|
||||
fail(adapterError('EXECUTOR_PROCESS_CANCELLED', `${label} execution was cancelled`));
|
||||
};
|
||||
if (signal?.aborted) onAbort();
|
||||
else signal?.addEventListener?.('abort', onAbort, { once: true });
|
||||
child.stdout?.on?.('data', (chunk) => {
|
||||
stdout = appendBounded(stdout, chunk, outputLimit);
|
||||
void emit('executor_job_output_chunk', {
|
||||
adapter: executorId,
|
||||
stream: 'stdout',
|
||||
bytes: Buffer.byteLength(chunk),
|
||||
});
|
||||
});
|
||||
child.stderr?.on?.('data', (chunk) => {
|
||||
stderr = appendBounded(stderr, chunk, outputLimit);
|
||||
void emit('executor_job_output_chunk', {
|
||||
adapter: executorId,
|
||||
stream: 'stderr',
|
||||
bytes: Buffer.byteLength(chunk),
|
||||
});
|
||||
});
|
||||
child.once?.('error', (error) => {
|
||||
fail(adapterError(
|
||||
'EXECUTOR_PROCESS_START_FAILED',
|
||||
`${label} failed to start: ${error.message}`,
|
||||
{ retryable: true, cause: error },
|
||||
));
|
||||
});
|
||||
child.once?.('exit', async (code, exitSignal) => {
|
||||
if (settled) return;
|
||||
if (code !== 0) {
|
||||
fail(adapterError(
|
||||
'EXECUTOR_PROCESS_FAILED',
|
||||
`${label} exited with ${exitSignal ?? code}: ${stderr.slice(-1024)}`,
|
||||
));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const artifactRefs = await artifactCollector({
|
||||
executor: executorId,
|
||||
request,
|
||||
cwd,
|
||||
stdout,
|
||||
stderr,
|
||||
});
|
||||
settled = true;
|
||||
cleanup();
|
||||
resolve({
|
||||
outcome: 'completed',
|
||||
summary: `${label} completed successfully`,
|
||||
artifactRefs,
|
||||
metrics: {
|
||||
stdoutBytes: Buffer.byteLength(stdout),
|
||||
stderrBytes: Buffer.byteLength(stderr),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
fail(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
id: executorId,
|
||||
label,
|
||||
kind: 'isolated-process',
|
||||
enabled: true,
|
||||
dispatchImplemented: true,
|
||||
capabilities: ['code-edit', 'cancel', 'artifact-refs', 'workspace-alias'],
|
||||
submit,
|
||||
async cancel(jobId) {
|
||||
const child = active.get(jobId);
|
||||
if (!child) return false;
|
||||
child.kill('SIGTERM');
|
||||
return true;
|
||||
},
|
||||
async getState(jobId) {
|
||||
return { active: active.has(jobId) };
|
||||
},
|
||||
async *streamEvents() {
|
||||
throw adapterError(
|
||||
'EXECUTOR_PROCESS_STREAM_PUSH_ONLY',
|
||||
'Process events are delivered through submit emit callbacks',
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createAiderExecutorAdapter(options = {}) {
|
||||
return createProcessExecutorAdapter({
|
||||
id: 'aider',
|
||||
label: 'Aider',
|
||||
buildArgs: (instruction) => ['--yes-always', '--message', instruction],
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
export function createOpenHandsExecutorAdapter(options = {}) {
|
||||
return createProcessExecutorAdapter({
|
||||
id: 'openhands',
|
||||
label: 'OpenHands',
|
||||
buildArgs: (instruction) => ['--headless', '--task', instruction],
|
||||
...options,
|
||||
});
|
||||
}
|
||||
|
||||
export const executorAdapterInternals = {
|
||||
DEFAULT_ALLOWED_ENV,
|
||||
DEFAULT_OUTPUT_LIMIT,
|
||||
EventEmitter,
|
||||
adapterError,
|
||||
appendBounded,
|
||||
isLoopbackHostname,
|
||||
normalizeBaseUrl,
|
||||
parseSse,
|
||||
resolveWorkspace,
|
||||
terminalGoosedEvent,
|
||||
};
|
||||
@@ -0,0 +1,162 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
createAiderExecutorAdapter,
|
||||
createGoosedExecutorAdapter,
|
||||
createOpenHandsExecutorAdapter,
|
||||
} from './executor-adapters.mjs';
|
||||
|
||||
function request(executor = 'goosed') {
|
||||
return {
|
||||
executor,
|
||||
task: {
|
||||
instruction: 'Fix the validation',
|
||||
workspaceRef: { kind: 'workspace-alias', id: 'canary' },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sseResponse(events) {
|
||||
const encoder = new TextEncoder();
|
||||
const body = new ReadableStream({
|
||||
start(controller) {
|
||||
for (const event of events) {
|
||||
controller.enqueue(encoder.encode(
|
||||
`event: ${event.type}\ndata: ${JSON.stringify(event.data)}\n\n`,
|
||||
));
|
||||
}
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
return new Response(body, {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/event-stream' },
|
||||
});
|
||||
}
|
||||
|
||||
test('Goosed adapter uses the current session API, streams bounded metadata and returns refs', async () => {
|
||||
const calls = [];
|
||||
const emitted = [];
|
||||
const adapter = createGoosedExecutorAdapter({
|
||||
baseUrl: 'http://127.0.0.1:18006',
|
||||
secret: 'secret',
|
||||
workspaceResolver: async () => '/tmp/canary',
|
||||
fetchImpl: async (url, init) => {
|
||||
calls.push({ url, init });
|
||||
if (url.endsWith('/agent/start')) {
|
||||
return Response.json({ id: 'session-1' });
|
||||
}
|
||||
if (url.endsWith('/events')) {
|
||||
return sseResponse([
|
||||
{ type: 'message', data: { type: 'message', secret: 'not-projected' } },
|
||||
{ type: 'finish', data: { type: 'finish' } },
|
||||
]);
|
||||
}
|
||||
if (url.endsWith('/reply')) return Response.json({ ok: true });
|
||||
throw new Error(`Unexpected URL: ${url}`);
|
||||
},
|
||||
});
|
||||
const result = await adapter.submit(request(), {
|
||||
jobId: 'job-1',
|
||||
emit: async (type, data) => emitted.push({ type, data }),
|
||||
});
|
||||
assert.deepEqual(result.artifactRefs, [{ kind: 'goosed-session', id: 'session-1' }]);
|
||||
assert.equal(result.metrics.eventCount, 2);
|
||||
assert.equal(calls[0].url, 'http://127.0.0.1:18006/agent/start');
|
||||
assert.equal(calls[0].init.headers.authorization, 'Bearer secret');
|
||||
assert.equal(JSON.parse(calls[0].init.body).working_dir, '/tmp/canary');
|
||||
assert.equal(
|
||||
JSON.parse(calls.find((call) => call.url.endsWith('/reply')).init.body)
|
||||
.user_message.content[0].text,
|
||||
'Fix the validation',
|
||||
);
|
||||
assert.deepEqual(
|
||||
emitted.map((event) => event.type),
|
||||
[
|
||||
'executor_job_session_created',
|
||||
'executor_job_adapter_event',
|
||||
'executor_job_adapter_event',
|
||||
],
|
||||
);
|
||||
assert.equal(JSON.stringify(emitted).includes('not-projected'), false);
|
||||
});
|
||||
|
||||
test('Goosed adapter rejects remote targets unless explicitly allowed', () => {
|
||||
assert.throws(
|
||||
() => createGoosedExecutorAdapter({ baseUrl: 'https://executor.example.com' }),
|
||||
/explicit allowRemote/,
|
||||
);
|
||||
assert.doesNotThrow(
|
||||
() => createGoosedExecutorAdapter({
|
||||
baseUrl: 'https://executor.example.com',
|
||||
allowRemote: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
test('Aider process adapter resolves a workspace alias and never invokes a shell', async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-executor-'));
|
||||
const workspace = path.join(root, 'canary');
|
||||
await fs.mkdir(workspace);
|
||||
const spawnCalls = [];
|
||||
const emitted = [];
|
||||
const spawnImpl = (command, args, options) => {
|
||||
spawnCalls.push({ command, args, options });
|
||||
const child = new EventEmitter();
|
||||
child.stdout = new EventEmitter();
|
||||
child.stderr = new EventEmitter();
|
||||
child.kill = () => {};
|
||||
queueMicrotask(() => {
|
||||
child.stdout.emit('data', Buffer.from('done'));
|
||||
child.emit('exit', 0, null);
|
||||
});
|
||||
return child;
|
||||
};
|
||||
try {
|
||||
const adapter = createAiderExecutorAdapter({
|
||||
command: '/usr/local/bin/aider',
|
||||
allowedRoot: root,
|
||||
workspaceAliases: { canary: workspace },
|
||||
spawnImpl,
|
||||
});
|
||||
const result = await adapter.submit(request('aider'), {
|
||||
jobId: 'job-1',
|
||||
emit: async (type, data) => emitted.push({ type, data }),
|
||||
});
|
||||
assert.equal(result.outcome, 'completed');
|
||||
assert.deepEqual(result.artifactRefs, [{ kind: 'workspace-alias', id: 'canary' }]);
|
||||
assert.equal(spawnCalls[0].options.cwd, await fs.realpath(workspace));
|
||||
assert.equal(spawnCalls[0].options.shell, false);
|
||||
assert.deepEqual(
|
||||
spawnCalls[0].args,
|
||||
['--yes-always', '--message', 'Fix the validation'],
|
||||
);
|
||||
assert.equal(emitted[0].data.stream, 'stdout');
|
||||
assert.equal('HOME' in spawnCalls[0].options.env, false);
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenHands adapter refuses workspace aliases outside its isolated root', async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-executor-root-'));
|
||||
const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-executor-outside-'));
|
||||
try {
|
||||
const adapter = createOpenHandsExecutorAdapter({
|
||||
command: '/usr/local/bin/openhands',
|
||||
allowedRoot: root,
|
||||
workspaceAliases: { canary: outside },
|
||||
});
|
||||
await assert.rejects(
|
||||
adapter.submit(request('openhands'), { jobId: 'job-1' }),
|
||||
(error) => error.code === 'EXECUTOR_WORKSPACE_OUTSIDE_ROOT',
|
||||
);
|
||||
} finally {
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
await fs.rm(outside, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,102 @@
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.floor(number)));
|
||||
}
|
||||
|
||||
function normalizeList(value) {
|
||||
const items = Array.isArray(value)
|
||||
? value
|
||||
: String(value ?? '').split(',');
|
||||
return [...new Set(items.map((item) => String(item).trim()).filter(Boolean))];
|
||||
}
|
||||
|
||||
export function normalizeExecutorAdmissionConfig(input = {}) {
|
||||
return {
|
||||
tenantAllowlist: normalizeList(input.tenantAllowlist),
|
||||
userAllowlist: normalizeList(input.userAllowlist),
|
||||
workspaceKindAllowlist: normalizeList(
|
||||
input.workspaceKindAllowlist ?? ['workspace-alias'],
|
||||
),
|
||||
maxGlobalActive: clampInteger(input.maxGlobalActive, 20, 1, 10_000),
|
||||
maxTenantActive: clampInteger(input.maxTenantActive, 5, 1, 1_000),
|
||||
maxUserActive: clampInteger(input.maxUserActive, 2, 1, 100),
|
||||
maxTenantPerMinute: clampInteger(input.maxTenantPerMinute, 30, 1, 100_000),
|
||||
maxUserPerMinute: clampInteger(input.maxUserPerMinute, 10, 1, 10_000),
|
||||
};
|
||||
}
|
||||
|
||||
export function createExecutorAdmissionPolicy({
|
||||
store,
|
||||
config = {},
|
||||
nowMs = Date.now,
|
||||
} = {}) {
|
||||
if (typeof store?.getAdmissionStats !== 'function') {
|
||||
throw new Error('Executor admission policy requires store.getAdmissionStats');
|
||||
}
|
||||
const normalized = normalizeExecutorAdmissionConfig(config);
|
||||
|
||||
function denied(reason, details = {}) {
|
||||
return {
|
||||
version: 'executor-admission-v1',
|
||||
allowed: false,
|
||||
reason,
|
||||
details,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
config: normalized,
|
||||
async evaluate(request) {
|
||||
const tenantId = request.subject?.tenantId;
|
||||
const userId = request.subject?.userId;
|
||||
const workspaceKind = request.task?.workspaceRef?.kind;
|
||||
if (normalized.tenantAllowlist.length && !normalized.tenantAllowlist.includes(tenantId)) {
|
||||
return denied('executor_tenant_not_allowed');
|
||||
}
|
||||
if (normalized.userAllowlist.length && !normalized.userAllowlist.includes(userId)) {
|
||||
return denied('executor_user_not_allowed');
|
||||
}
|
||||
if (!normalized.workspaceKindAllowlist.includes(workspaceKind)) {
|
||||
return denied('executor_workspace_kind_not_allowed', { workspaceKind });
|
||||
}
|
||||
const stats = await store.getAdmissionStats({
|
||||
tenantId,
|
||||
userId,
|
||||
now: nowMs(),
|
||||
windowMs: 60_000,
|
||||
});
|
||||
const checks = [
|
||||
['globalActive', normalized.maxGlobalActive, 'executor_global_concurrency_limited'],
|
||||
['tenantActive', normalized.maxTenantActive, 'executor_tenant_concurrency_limited'],
|
||||
['userActive', normalized.maxUserActive, 'executor_user_concurrency_limited'],
|
||||
['tenantRecent', normalized.maxTenantPerMinute, 'executor_tenant_rate_limited'],
|
||||
['userRecent', normalized.maxUserPerMinute, 'executor_user_rate_limited'],
|
||||
];
|
||||
for (const [field, limit, reason] of checks) {
|
||||
if (stats[field] >= limit) return denied(reason, { current: stats[field], limit });
|
||||
}
|
||||
return {
|
||||
version: 'executor-admission-v1',
|
||||
allowed: true,
|
||||
reason: 'executor_admitted',
|
||||
details: {
|
||||
globalActive: stats.globalActive,
|
||||
tenantActive: stats.tenantActive,
|
||||
userActive: stats.userActive,
|
||||
},
|
||||
};
|
||||
},
|
||||
status() {
|
||||
return {
|
||||
version: 'executor-admission-policy-v1',
|
||||
...normalized,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const executorAdmissionPolicyInternals = {
|
||||
clampInteger,
|
||||
normalizeList,
|
||||
};
|
||||
@@ -0,0 +1,67 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { createExecutorAdmissionPolicy } from './executor-admission-policy.mjs';
|
||||
import {
|
||||
createExecutorGateway,
|
||||
createInMemoryExecutorJobStore,
|
||||
createPhase5ExecutorAdapterRegistry,
|
||||
} from './executor-gateway.mjs';
|
||||
|
||||
function request(overrides = {}) {
|
||||
return {
|
||||
jobId: 'job-1',
|
||||
idempotencyKey: 'idem-1',
|
||||
executor: 'goosed',
|
||||
task: {
|
||||
instruction: 'do it',
|
||||
workspaceRef: { kind: 'workspace-alias', id: 'canary' },
|
||||
},
|
||||
subject: { tenantId: 'tenant-1', userId: 'user-1' },
|
||||
authorization: { executionAllowed: true },
|
||||
policy: { sideEffectsAllowed: true },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('admission policy enforces allowlists, workspace kinds and durable-store quotas', async () => {
|
||||
const store = createInMemoryExecutorJobStore();
|
||||
const policy = createExecutorAdmissionPolicy({
|
||||
store,
|
||||
config: {
|
||||
tenantAllowlist: ['tenant-1'],
|
||||
userAllowlist: ['user-1'],
|
||||
maxUserActive: 1,
|
||||
},
|
||||
});
|
||||
assert.equal((await policy.evaluate(request())).allowed, true);
|
||||
assert.equal(
|
||||
(await policy.evaluate(request({
|
||||
subject: { tenantId: 'tenant-2', userId: 'user-1' },
|
||||
}))).reason,
|
||||
'executor_tenant_not_allowed',
|
||||
);
|
||||
assert.equal(
|
||||
(await policy.evaluate(request({
|
||||
task: {
|
||||
instruction: 'do it',
|
||||
workspaceRef: { kind: 'absolute-path', id: '/tmp/repo' },
|
||||
},
|
||||
}))).reason,
|
||||
'executor_workspace_kind_not_allowed',
|
||||
);
|
||||
|
||||
const gateway = createExecutorGateway({
|
||||
store,
|
||||
registry: createPhase5ExecutorAdapterRegistry({ enabledExecutors: ['goosed'] }),
|
||||
executionEnabled: true,
|
||||
admissionPolicy: policy,
|
||||
});
|
||||
const first = await gateway.createJob(request());
|
||||
assert.equal(first.job.status, 'queued');
|
||||
const second = await gateway.createJob(request({
|
||||
jobId: 'job-2',
|
||||
idempotencyKey: 'idem-2',
|
||||
}));
|
||||
assert.equal(second.job.status, 'blocked');
|
||||
assert.equal(second.job.reason, 'executor_user_concurrency_limited');
|
||||
});
|
||||
@@ -4,7 +4,7 @@ import { normalizeWorkflowEngineId } from './contracts.mjs';
|
||||
const EXECUTOR_JOB_REQUEST_VERSION = 'executor-job-request-v1';
|
||||
const EXECUTOR_JOB_STATE_VERSION = 'executor-job-state-v1';
|
||||
const EXECUTOR_JOB_EVENT_VERSION = 'executor-job-event-v1';
|
||||
const EXECUTOR_DISPATCH_IMPLEMENTED = false;
|
||||
const EXECUTOR_DISPATCH_IMPLEMENTED = true;
|
||||
const MAX_INSTRUCTION_CHARACTERS = 32_000;
|
||||
const MAX_EVENT_PAGE_SIZE = 500;
|
||||
const TERMINAL_JOB_STATUSES = new Set([
|
||||
@@ -167,6 +167,7 @@ export function normalizeExecutorJobRequest(input = {}) {
|
||||
timeoutMs: clampInteger(source.controls?.timeoutMs, 15 * 60_000, 500, 60 * 60_000),
|
||||
cancellationAllowed: source.controls?.cancellationAllowed !== false,
|
||||
fallbackExecutor: normalizeWorkflowEngineId(source.controls?.fallbackExecutor, ''),
|
||||
maxAttempts: clampInteger(source.controls?.maxAttempts, 3, 1, 20),
|
||||
},
|
||||
metadata: source.metadata && typeof source.metadata === 'object'
|
||||
? clone(source.metadata)
|
||||
@@ -252,6 +253,34 @@ export function createDisabledExecutorAdapter(descriptor) {
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerQueueExecutorAdapter(descriptor, { enabled = false } = {}) {
|
||||
const id = normalizeWorkflowEngineId(descriptor?.id, '');
|
||||
if (!id) throw new Error('Worker queue executor adapter requires a valid id');
|
||||
const reject = () => {
|
||||
throw gatewayError(
|
||||
'EXECUTOR_WORKER_CLAIM_REQUIRED',
|
||||
`Executor adapter ${id} is dispatched through the worker claim protocol`,
|
||||
);
|
||||
};
|
||||
return {
|
||||
id,
|
||||
label: descriptor.label ?? id,
|
||||
kind: descriptor.kind ?? 'worker',
|
||||
enabled,
|
||||
dispatchImplemented: true,
|
||||
status: enabled ? 'worker-queue' : 'disabled',
|
||||
capabilities: Array.isArray(descriptor.capabilities)
|
||||
? [...descriptor.capabilities, 'lease-fencing']
|
||||
: ['lease-fencing'],
|
||||
submit: reject,
|
||||
cancel: reject,
|
||||
getState: reject,
|
||||
async *streamEvents() {
|
||||
reject();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createPhase3ExecutorAdapterRegistry() {
|
||||
return createExecutorAdapterRegistry(
|
||||
PHASE3_EXECUTOR_DESCRIPTORS.map(createDisabledExecutorAdapter),
|
||||
@@ -262,10 +291,49 @@ export function listPhase3ExecutorAdapters() {
|
||||
return createPhase3ExecutorAdapterRegistry().list();
|
||||
}
|
||||
|
||||
export function createPhase5ExecutorAdapterRegistry({ enabledExecutors = [] } = {}) {
|
||||
const enabled = new Set(
|
||||
(Array.isArray(enabledExecutors) ? enabledExecutors : [])
|
||||
.map((id) => normalizeWorkflowEngineId(id, ''))
|
||||
.filter(Boolean),
|
||||
);
|
||||
return createExecutorAdapterRegistry(
|
||||
PHASE3_EXECUTOR_DESCRIPTORS.map((descriptor) => (
|
||||
createWorkerQueueExecutorAdapter(descriptor, { enabled: enabled.has(descriptor.id) })
|
||||
)),
|
||||
);
|
||||
}
|
||||
|
||||
export function listPhase5ExecutorAdapters(options = {}) {
|
||||
return createPhase5ExecutorAdapterRegistry(options).list();
|
||||
}
|
||||
|
||||
export function projectExecutorJobForRead(record) {
|
||||
if (!record) return null;
|
||||
const {
|
||||
request: _request,
|
||||
requestFingerprint: _requestFingerprint,
|
||||
lease,
|
||||
...safe
|
||||
} = clone(record);
|
||||
return {
|
||||
...safe,
|
||||
lease: lease
|
||||
? {
|
||||
workerId: lease.workerId ?? null,
|
||||
acquiredAt: lease.acquiredAt ?? null,
|
||||
heartbeatAt: lease.heartbeatAt ?? null,
|
||||
expiresAt: lease.expiresAt ?? null,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function createInMemoryExecutorJobStore() {
|
||||
const byId = new Map();
|
||||
const byIdempotencyKey = new Map();
|
||||
const eventsByJobId = new Map();
|
||||
const workersById = new Map();
|
||||
|
||||
function appendEvent(jobId, event) {
|
||||
if (!event) return;
|
||||
@@ -278,6 +346,14 @@ export function createInMemoryExecutorJobStore() {
|
||||
eventsByJobId.set(jobId, events);
|
||||
}
|
||||
|
||||
function transitionResult(applied, record, reason = null) {
|
||||
return {
|
||||
applied,
|
||||
record: record ? clone(record) : null,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
kind: 'memory',
|
||||
durable: false,
|
||||
@@ -307,6 +383,160 @@ export function createInMemoryExecutorJobStore() {
|
||||
appendEvent(record.jobId, event);
|
||||
return clone(record);
|
||||
},
|
||||
async claimNext({
|
||||
workerId,
|
||||
executors,
|
||||
now,
|
||||
leaseToken,
|
||||
leaseDurationMs,
|
||||
eventFactory,
|
||||
}) {
|
||||
const allowedExecutors = new Set(executors);
|
||||
const candidate = [...byId.values()]
|
||||
.filter((record) => (
|
||||
allowedExecutors.has(record.executor)
|
||||
&& (
|
||||
record.status === 'queued'
|
||||
|| (
|
||||
record.status === 'retryable'
|
||||
&& Number(record.nextAttemptAt ?? 0) <= now
|
||||
)
|
||||
)
|
||||
))
|
||||
.sort((left, right) => (
|
||||
Number(left.createdAt ?? 0) - Number(right.createdAt ?? 0)
|
||||
|| String(left.jobId).localeCompare(String(right.jobId))
|
||||
))[0];
|
||||
if (!candidate) return null;
|
||||
const updated = {
|
||||
...clone(candidate),
|
||||
status: 'leased',
|
||||
reason: null,
|
||||
attempts: Number(candidate.attempts ?? 0) + 1,
|
||||
nextAttemptAt: null,
|
||||
lease: {
|
||||
token: leaseToken,
|
||||
workerId,
|
||||
acquiredAt: now,
|
||||
heartbeatAt: now,
|
||||
expiresAt: now + leaseDurationMs,
|
||||
},
|
||||
updatedAt: now,
|
||||
completedAt: null,
|
||||
};
|
||||
byId.set(updated.jobId, clone(updated));
|
||||
appendEvent(
|
||||
updated.jobId,
|
||||
typeof eventFactory === 'function' ? eventFactory(updated, candidate) : null,
|
||||
);
|
||||
return clone(updated);
|
||||
},
|
||||
async transition(jobId, {
|
||||
expectedStatuses = [],
|
||||
leaseToken = null,
|
||||
updater,
|
||||
eventFactory = null,
|
||||
} = {}) {
|
||||
const normalizedJobId = String(jobId ?? '').trim();
|
||||
const current = byId.get(normalizedJobId);
|
||||
if (!current) return transitionResult(false, null, 'not_found');
|
||||
if (expectedStatuses.length && !expectedStatuses.includes(current.status)) {
|
||||
return transitionResult(false, current, 'status_mismatch');
|
||||
}
|
||||
if (leaseToken != null && current.lease?.token !== leaseToken) {
|
||||
return transitionResult(false, current, 'lease_mismatch');
|
||||
}
|
||||
const updated = updater(clone(current));
|
||||
if (!updated || updated.jobId !== current.jobId) {
|
||||
throw new Error('Executor job transition must preserve job identity');
|
||||
}
|
||||
byId.set(normalizedJobId, clone(updated));
|
||||
appendEvent(
|
||||
normalizedJobId,
|
||||
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
|
||||
);
|
||||
return transitionResult(true, updated);
|
||||
},
|
||||
async listExpiredLeases({ now, limit = 100 } = {}) {
|
||||
return [...byId.values()]
|
||||
.filter((record) => (
|
||||
['leased', 'running'].includes(record.status)
|
||||
&& Number(record.lease?.expiresAt ?? 0) <= now
|
||||
))
|
||||
.sort((left, right) => (
|
||||
Number(left.lease?.expiresAt ?? 0) - Number(right.lease?.expiresAt ?? 0)
|
||||
))
|
||||
.slice(0, clampInteger(limit, 100, 1, 500))
|
||||
.map(clone);
|
||||
},
|
||||
async getQueueStats({ now = Date.now() } = {}) {
|
||||
const records = [...byId.values()];
|
||||
const counts = {};
|
||||
for (const record of records) {
|
||||
counts[record.status] = Number(counts[record.status] ?? 0) + 1;
|
||||
}
|
||||
return {
|
||||
total: records.length,
|
||||
counts,
|
||||
claimable: records.filter((record) => (
|
||||
record.status === 'queued'
|
||||
|| (
|
||||
record.status === 'retryable'
|
||||
&& Number(record.nextAttemptAt ?? 0) <= now
|
||||
)
|
||||
)).length,
|
||||
expiredLeases: records.filter((record) => (
|
||||
['leased', 'running'].includes(record.status)
|
||||
&& Number(record.lease?.expiresAt ?? 0) <= now
|
||||
)).length,
|
||||
};
|
||||
},
|
||||
async getAdmissionStats({
|
||||
tenantId = null,
|
||||
userId = null,
|
||||
now = Date.now(),
|
||||
windowMs = 60_000,
|
||||
} = {}) {
|
||||
const activeStatuses = new Set(['queued', 'retryable', 'leased', 'running']);
|
||||
const records = [...byId.values()];
|
||||
const matchesTenant = (record) => (
|
||||
tenantId != null && record.request?.subject?.tenantId === tenantId
|
||||
);
|
||||
const matchesUser = (record) => (
|
||||
userId != null && record.request?.subject?.userId === userId
|
||||
);
|
||||
return {
|
||||
globalActive: records.filter((record) => activeStatuses.has(record.status)).length,
|
||||
tenantActive: tenantId == null
|
||||
? 0
|
||||
: records.filter((record) => activeStatuses.has(record.status) && matchesTenant(record)).length,
|
||||
userActive: userId == null
|
||||
? 0
|
||||
: records.filter((record) => activeStatuses.has(record.status) && matchesUser(record)).length,
|
||||
tenantRecent: tenantId == null
|
||||
? 0
|
||||
: records.filter((record) => (
|
||||
matchesTenant(record) && Number(record.createdAt ?? 0) >= now - windowMs
|
||||
)).length,
|
||||
userRecent: userId == null
|
||||
? 0
|
||||
: records.filter((record) => (
|
||||
matchesUser(record) && Number(record.createdAt ?? 0) >= now - windowMs
|
||||
)).length,
|
||||
};
|
||||
},
|
||||
async recordWorkerHeartbeat(record) {
|
||||
workersById.set(record.workerId, clone(record));
|
||||
return clone(record);
|
||||
},
|
||||
async listWorkers({ now = Date.now(), staleAfterMs = 60_000 } = {}) {
|
||||
return [...workersById.values()]
|
||||
.sort((left, right) => Number(right.lastSeenAt) - Number(left.lastSeenAt))
|
||||
.map((worker) => ({
|
||||
...clone(worker),
|
||||
stale: Number(worker.lastSeenAt ?? 0) < now - staleAfterMs,
|
||||
}));
|
||||
},
|
||||
async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
|
||||
const cursor = Math.max(0, Number(after) || 0);
|
||||
const pageSize = clampInteger(limit, MAX_EVENT_PAGE_SIZE, 1, MAX_EVENT_PAGE_SIZE);
|
||||
@@ -336,6 +566,8 @@ export function createExecutorGateway({
|
||||
registry = createPhase3ExecutorAdapterRegistry(),
|
||||
store = createInMemoryExecutorJobStore(),
|
||||
nowMs = Date.now,
|
||||
executionEnabled = false,
|
||||
admissionPolicy = null,
|
||||
} = {}) {
|
||||
if (!registry?.has || !registry?.list) {
|
||||
throw new Error('Executor Gateway requires an adapter registry');
|
||||
@@ -352,6 +584,7 @@ export function createExecutorGateway({
|
||||
&& fallbackAdapter?.dispatchImplemented === true;
|
||||
const gates = {
|
||||
implementation: EXECUTOR_DISPATCH_IMPLEMENTED,
|
||||
serviceEnabled: executionEnabled === true,
|
||||
adapterRegistered: Boolean(adapter),
|
||||
adapterEnabled: adapter?.enabled === true,
|
||||
executionAuthorized: request.authorization.executionAllowed,
|
||||
@@ -367,9 +600,21 @@ export function createExecutorGateway({
|
||||
fallbackRegistered,
|
||||
fallbackAvailable,
|
||||
dispatchAllowed: Object.values(gates).every(Boolean),
|
||||
reason: EXECUTOR_DISPATCH_IMPLEMENTED
|
||||
? 'executor_adapter_unavailable'
|
||||
: 'executor_dispatch_not_implemented',
|
||||
reason: !EXECUTOR_DISPATCH_IMPLEMENTED
|
||||
? 'executor_dispatch_not_implemented'
|
||||
: executionEnabled !== true
|
||||
? 'executor_execution_disabled'
|
||||
: !adapter
|
||||
? 'executor_adapter_unregistered'
|
||||
: adapter.enabled !== true
|
||||
? 'executor_adapter_disabled'
|
||||
: !request.authorization.executionAllowed
|
||||
? 'executor_execution_unauthorized'
|
||||
: !request.policy.sideEffectsAllowed
|
||||
? 'executor_side_effects_not_allowed'
|
||||
: !request.task.workspaceRef
|
||||
? 'executor_workspace_required'
|
||||
: 'queued_for_worker',
|
||||
gates,
|
||||
};
|
||||
}
|
||||
@@ -396,40 +641,56 @@ export function createExecutorGateway({
|
||||
}
|
||||
return { created: false, job: existing };
|
||||
}
|
||||
const decision = previewNormalized(request);
|
||||
let decision = previewNormalized(request);
|
||||
if (decision.dispatchAllowed && admissionPolicy?.evaluate) {
|
||||
const admission = await admissionPolicy.evaluate(request);
|
||||
if (!admission.allowed) {
|
||||
decision = {
|
||||
...decision,
|
||||
dispatchAllowed: false,
|
||||
reason: admission.reason,
|
||||
admission,
|
||||
};
|
||||
} else {
|
||||
decision = { ...decision, admission };
|
||||
}
|
||||
}
|
||||
const now = nowMs();
|
||||
const dispatchAllowed = decision.dispatchAllowed;
|
||||
const record = {
|
||||
version: EXECUTOR_JOB_STATE_VERSION,
|
||||
jobId: request.jobId,
|
||||
idempotencyKey: request.idempotencyKey,
|
||||
requestFingerprint,
|
||||
executor: request.executor,
|
||||
status: 'blocked',
|
||||
status: dispatchAllowed ? 'queued' : 'blocked',
|
||||
reason: decision.reason,
|
||||
attempts: 0,
|
||||
dispatchAllowed: false,
|
||||
maxAttempts: request.controls.maxAttempts,
|
||||
dispatchAllowed,
|
||||
cancellationAllowed: request.controls.cancellationAllowed,
|
||||
timeoutMs: request.controls.timeoutMs,
|
||||
fallbackExecutor: decision.fallbackExecutor,
|
||||
fallbackRegistered: decision.fallbackRegistered,
|
||||
fallbackAvailable: decision.fallbackAvailable,
|
||||
decision,
|
||||
request: dispatchAllowed ? request : null,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
startedAt: null,
|
||||
completedAt: now,
|
||||
completedAt: dispatchAllowed ? null : now,
|
||||
};
|
||||
const result = await store.createIfAbsent(record, {
|
||||
initialEvent: buildExecutorJobEvent({
|
||||
jobId: request.jobId,
|
||||
sequence: 1,
|
||||
type: 'executor_job_blocked',
|
||||
type: dispatchAllowed ? 'executor_job_queued' : 'executor_job_blocked',
|
||||
timestamp: now,
|
||||
data: {
|
||||
executor: request.executor,
|
||||
status: record.status,
|
||||
reason: record.reason,
|
||||
dispatchAllowed: false,
|
||||
dispatchAllowed,
|
||||
},
|
||||
}),
|
||||
});
|
||||
@@ -451,32 +712,42 @@ export function createExecutorGateway({
|
||||
if (current.status === 'cancelled' || !current.cancellationAllowed) return current;
|
||||
if (TERMINAL_JOB_STATUSES.has(current.status) && current.status !== 'blocked') return current;
|
||||
const now = nowMs();
|
||||
const updated = {
|
||||
...current,
|
||||
const applyCancellation = (record) => ({
|
||||
...record,
|
||||
status: 'cancelled',
|
||||
reason: String(reason).slice(0, 256),
|
||||
lease: null,
|
||||
updatedAt: now,
|
||||
completedAt: now,
|
||||
};
|
||||
return store.update(updated, {
|
||||
event: buildExecutorJobEvent({
|
||||
jobId: current.jobId,
|
||||
type: 'executor_job_cancelled',
|
||||
timestamp: now,
|
||||
data: {
|
||||
status: updated.status,
|
||||
reason: updated.reason,
|
||||
},
|
||||
}),
|
||||
});
|
||||
const eventForCancellation = (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_cancelled',
|
||||
timestamp: now,
|
||||
data: {
|
||||
status: updated.status,
|
||||
reason: updated.reason,
|
||||
},
|
||||
});
|
||||
if (typeof store.transition === 'function') {
|
||||
const result = await store.transition(current.jobId, {
|
||||
expectedStatuses: ['blocked', 'queued', 'retryable', 'leased', 'running'],
|
||||
updater: applyCancellation,
|
||||
eventFactory: eventForCancellation,
|
||||
});
|
||||
return result.applied ? result.record : result.record;
|
||||
}
|
||||
const updated = applyCancellation(current);
|
||||
return store.update(updated, { event: eventForCancellation(updated) });
|
||||
}
|
||||
|
||||
return {
|
||||
preview,
|
||||
createJob,
|
||||
cancel,
|
||||
getJob(jobId) {
|
||||
return store.getById(jobId);
|
||||
async getJob(jobId, { includeRequest = false } = {}) {
|
||||
const record = await store.getById(jobId);
|
||||
return includeRequest ? record : projectExecutorJobForRead(record);
|
||||
},
|
||||
async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
|
||||
const normalizedJobId = String(jobId ?? '').trim();
|
||||
@@ -501,7 +772,7 @@ export function createExecutorGateway({
|
||||
return {
|
||||
version: 'executor-gateway-status-v1',
|
||||
dispatchImplemented: EXECUTOR_DISPATCH_IMPLEMENTED,
|
||||
executionEnabled: false,
|
||||
executionEnabled: executionEnabled === true,
|
||||
store: {
|
||||
kind: String(store.kind ?? (store.durable ? 'durable' : 'memory')),
|
||||
durable: store.durable === true,
|
||||
@@ -522,5 +793,6 @@ export const executorGatewayInternals = {
|
||||
PHASE3_EXECUTOR_DESCRIPTORS,
|
||||
TERMINAL_JOB_STATUSES,
|
||||
fingerprint,
|
||||
projectExecutorJobForRead,
|
||||
stableValue,
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
createExecutorGateway,
|
||||
createInMemoryExecutorJobStore,
|
||||
createPhase3ExecutorAdapterRegistry,
|
||||
createPhase5ExecutorAdapterRegistry,
|
||||
normalizeExecutorJobRequest,
|
||||
} from './executor-gateway.mjs';
|
||||
|
||||
@@ -107,14 +108,15 @@ test('Executor Gateway records a blocked idempotent job without calling an adapt
|
||||
assert.equal(preview.fallbackRegistered, true);
|
||||
assert.equal(preview.fallbackAvailable, false);
|
||||
assert.equal(preview.dispatchAllowed, false);
|
||||
assert.equal(preview.gates.implementation, false);
|
||||
assert.equal(preview.gates.implementation, true);
|
||||
assert.equal(preview.gates.serviceEnabled, false);
|
||||
assert.equal(preview.gates.adapterEnabled, false);
|
||||
|
||||
const first = await gateway.createJob(request());
|
||||
assert.equal(first.created, true);
|
||||
assert.equal(first.job.status, 'blocked');
|
||||
assert.equal(first.job.attempts, 0);
|
||||
assert.equal(first.job.reason, 'executor_dispatch_not_implemented');
|
||||
assert.equal(first.job.reason, 'executor_execution_disabled');
|
||||
|
||||
now = 2000;
|
||||
const repeated = await gateway.createJob(request());
|
||||
@@ -160,10 +162,31 @@ test('Executor Gateway cancellation is idempotent and never invokes a disabled a
|
||||
assert.equal(await gateway.listEvents('missing-job'), null);
|
||||
});
|
||||
|
||||
test('Executor Gateway status remains hard-disabled with a non-durable reference store', () => {
|
||||
test('Executor Gateway reports implemented capability while execution remains disabled', () => {
|
||||
const status = createExecutorGateway().status();
|
||||
assert.equal(status.version, 'executor-gateway-status-v1');
|
||||
assert.equal(status.dispatchImplemented, false);
|
||||
assert.equal(status.dispatchImplemented, true);
|
||||
assert.equal(status.executionEnabled, false);
|
||||
assert.deepEqual(status.store, { kind: 'memory', durable: false });
|
||||
});
|
||||
|
||||
test('Executor Gateway queues only explicitly enabled and authorized worker jobs', async () => {
|
||||
const gateway = createExecutorGateway({
|
||||
registry: createPhase5ExecutorAdapterRegistry({ enabledExecutors: ['aider'] }),
|
||||
executionEnabled: true,
|
||||
});
|
||||
const created = await gateway.createJob(request());
|
||||
assert.equal(created.job.status, 'queued');
|
||||
assert.equal(created.job.reason, 'queued_for_worker');
|
||||
assert.equal(created.job.dispatchAllowed, true);
|
||||
assert.equal(created.job.maxAttempts, 3);
|
||||
assert.equal(created.job.request.task.instruction, 'Fix the login form validation');
|
||||
|
||||
const projected = await gateway.getJob('job-1');
|
||||
assert.equal('request' in projected, false);
|
||||
assert.equal('requestFingerprint' in projected, false);
|
||||
assert.deepEqual(
|
||||
(await gateway.listEvents('job-1')).events.map((event) => event.type),
|
||||
['executor_job_queued'],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ const { Pool } = pg;
|
||||
const DEFAULT_EXECUTOR_SCHEMA = 'memind_orchestrator';
|
||||
const EXECUTOR_JOB_TABLE = 'executor_jobs';
|
||||
const EXECUTOR_JOB_EVENT_TABLE = 'executor_job_events';
|
||||
const EXECUTOR_WORKER_TABLE = 'executor_workers';
|
||||
|
||||
function normalizeMode(value) {
|
||||
const mode = String(value ?? 'postgres').trim().toLowerCase();
|
||||
@@ -46,6 +47,31 @@ function eventWithSequence(event, jobId, sequence) {
|
||||
};
|
||||
}
|
||||
|
||||
async function appendEvent(client, eventTable, jobId, event) {
|
||||
if (!event) return null;
|
||||
const sequenceResult = await client.query(
|
||||
`SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence
|
||||
FROM ${eventTable}
|
||||
WHERE job_id = $1`,
|
||||
[jobId],
|
||||
);
|
||||
const sequence = Number(sequenceResult.rows?.[0]?.next_sequence ?? 1);
|
||||
const persistedEvent = eventWithSequence(event, jobId, sequence);
|
||||
await client.query(
|
||||
`INSERT INTO ${eventTable}
|
||||
(event_id, job_id, sequence, event_json, created_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5)`,
|
||||
[
|
||||
persistedEvent.eventId,
|
||||
jobId,
|
||||
sequence,
|
||||
JSON.stringify(persistedEvent),
|
||||
persistedEvent.timestamp,
|
||||
],
|
||||
);
|
||||
return persistedEvent;
|
||||
}
|
||||
|
||||
async function withTransaction(pool, operation) {
|
||||
if (typeof pool?.connect !== 'function') {
|
||||
throw new Error('PostgreSQL Executor Job Store requires transactional pool.connect');
|
||||
@@ -76,6 +102,7 @@ export function createPostgresExecutorJobStore({
|
||||
const normalizedSchema = normalizeSchema(schema);
|
||||
const table = `"${normalizedSchema}"."${EXECUTOR_JOB_TABLE}"`;
|
||||
const eventTable = `"${normalizedSchema}"."${EXECUTOR_JOB_EVENT_TABLE}"`;
|
||||
const workerTable = `"${normalizedSchema}"."${EXECUTOR_WORKER_TABLE}"`;
|
||||
|
||||
return {
|
||||
kind: 'postgres',
|
||||
@@ -136,6 +163,17 @@ export function createPostgresExecutorJobStore({
|
||||
)
|
||||
ON CONFLICT DO NOTHING
|
||||
`);
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS ${workerTable} (
|
||||
worker_id VARCHAR(128) PRIMARY KEY,
|
||||
state_json JSONB NOT NULL,
|
||||
last_seen_at BIGINT NOT NULL
|
||||
)
|
||||
`);
|
||||
await pool.query(`
|
||||
CREATE INDEX IF NOT EXISTS executor_workers_last_seen_idx
|
||||
ON ${workerTable} (last_seen_at DESC)
|
||||
`);
|
||||
},
|
||||
|
||||
async getById(jobId) {
|
||||
@@ -218,32 +256,242 @@ export function createPostgresExecutorJobStore({
|
||||
RETURNING state_json`,
|
||||
[record.jobId, JSON.stringify(record), record.updatedAt],
|
||||
);
|
||||
if (event) {
|
||||
const sequenceResult = await client.query(
|
||||
`SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence
|
||||
FROM ${eventTable}
|
||||
WHERE job_id = $1`,
|
||||
[record.jobId],
|
||||
);
|
||||
const sequence = Number(sequenceResult.rows?.[0]?.next_sequence ?? 1);
|
||||
const persistedEvent = eventWithSequence(event, record.jobId, sequence);
|
||||
await client.query(
|
||||
`INSERT INTO ${eventTable}
|
||||
(event_id, job_id, sequence, event_json, created_at)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5)`,
|
||||
[
|
||||
persistedEvent.eventId,
|
||||
record.jobId,
|
||||
sequence,
|
||||
JSON.stringify(persistedEvent),
|
||||
persistedEvent.timestamp,
|
||||
],
|
||||
);
|
||||
}
|
||||
await appendEvent(client, eventTable, record.jobId, event);
|
||||
return projectRow(result.rows?.[0]);
|
||||
});
|
||||
},
|
||||
|
||||
async claimNext({
|
||||
workerId,
|
||||
executors,
|
||||
now,
|
||||
leaseToken,
|
||||
leaseDurationMs,
|
||||
eventFactory,
|
||||
}) {
|
||||
return withTransaction(pool, async (client) => {
|
||||
const selected = await client.query(
|
||||
`SELECT state_json
|
||||
FROM ${table}
|
||||
WHERE (
|
||||
state_json->>'status' = 'queued'
|
||||
OR (
|
||||
state_json->>'status' = 'retryable'
|
||||
AND COALESCE((state_json->>'nextAttemptAt')::BIGINT, 0) <= $1
|
||||
)
|
||||
)
|
||||
AND state_json->>'executor' = ANY($2::text[])
|
||||
ORDER BY created_at ASC, job_id ASC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1`,
|
||||
[now, executors],
|
||||
);
|
||||
const current = projectRow(selected.rows?.[0]);
|
||||
if (!current) return null;
|
||||
const updated = {
|
||||
...current,
|
||||
status: 'leased',
|
||||
reason: null,
|
||||
attempts: Number(current.attempts ?? 0) + 1,
|
||||
nextAttemptAt: null,
|
||||
lease: {
|
||||
token: leaseToken,
|
||||
workerId,
|
||||
acquiredAt: now,
|
||||
heartbeatAt: now,
|
||||
expiresAt: now + leaseDurationMs,
|
||||
},
|
||||
updatedAt: now,
|
||||
completedAt: null,
|
||||
};
|
||||
const result = await client.query(
|
||||
`UPDATE ${table}
|
||||
SET state_json = $2::jsonb,
|
||||
updated_at = $3
|
||||
WHERE job_id = $1
|
||||
RETURNING state_json`,
|
||||
[updated.jobId, JSON.stringify(updated), updated.updatedAt],
|
||||
);
|
||||
await appendEvent(
|
||||
client,
|
||||
eventTable,
|
||||
updated.jobId,
|
||||
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
|
||||
);
|
||||
return projectRow(result.rows?.[0]);
|
||||
});
|
||||
},
|
||||
|
||||
async transition(jobId, {
|
||||
expectedStatuses = [],
|
||||
leaseToken = null,
|
||||
updater,
|
||||
eventFactory = null,
|
||||
} = {}) {
|
||||
return withTransaction(pool, async (client) => {
|
||||
const locked = await client.query(
|
||||
`SELECT state_json FROM ${table} WHERE job_id = $1 FOR UPDATE`,
|
||||
[String(jobId ?? '').trim()],
|
||||
);
|
||||
const current = projectRow(locked.rows?.[0]);
|
||||
if (!current) return { applied: false, record: null, reason: 'not_found' };
|
||||
if (expectedStatuses.length && !expectedStatuses.includes(current.status)) {
|
||||
return { applied: false, record: current, reason: 'status_mismatch' };
|
||||
}
|
||||
if (leaseToken != null && current.lease?.token !== leaseToken) {
|
||||
return { applied: false, record: current, reason: 'lease_mismatch' };
|
||||
}
|
||||
const updated = updater(structuredClone(current));
|
||||
if (!updated || updated.jobId !== current.jobId) {
|
||||
throw new Error('Executor job transition must preserve job identity');
|
||||
}
|
||||
const result = await client.query(
|
||||
`UPDATE ${table}
|
||||
SET state_json = $2::jsonb,
|
||||
updated_at = $3
|
||||
WHERE job_id = $1
|
||||
RETURNING state_json`,
|
||||
[current.jobId, JSON.stringify(updated), updated.updatedAt],
|
||||
);
|
||||
await appendEvent(
|
||||
client,
|
||||
eventTable,
|
||||
current.jobId,
|
||||
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
|
||||
);
|
||||
return {
|
||||
applied: true,
|
||||
record: projectRow(result.rows?.[0]),
|
||||
reason: null,
|
||||
};
|
||||
});
|
||||
},
|
||||
|
||||
async listExpiredLeases({ now, limit = 100 } = {}) {
|
||||
const pageSize = Math.min(500, Math.max(1, Number(limit) || 100));
|
||||
const result = await pool.query(
|
||||
`SELECT state_json
|
||||
FROM ${table}
|
||||
WHERE state_json->>'status' = ANY($1::text[])
|
||||
AND COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) <= $2
|
||||
ORDER BY COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) ASC
|
||||
LIMIT $3`,
|
||||
[['leased', 'running'], now, pageSize],
|
||||
);
|
||||
return (result.rows ?? []).map(projectRow).filter(Boolean);
|
||||
},
|
||||
|
||||
async getQueueStats({ now = Date.now() } = {}) {
|
||||
const result = await pool.query(
|
||||
`SELECT
|
||||
COUNT(*)::BIGINT AS total,
|
||||
COUNT(*) FILTER (
|
||||
WHERE state_json->>'status' = 'queued'
|
||||
OR (
|
||||
state_json->>'status' = 'retryable'
|
||||
AND COALESCE((state_json->>'nextAttemptAt')::BIGINT, 0) <= $1
|
||||
)
|
||||
)::BIGINT AS claimable,
|
||||
COUNT(*) FILTER (
|
||||
WHERE state_json->>'status' = ANY($2::text[])
|
||||
AND COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) <= $1
|
||||
)::BIGINT AS expired_leases,
|
||||
state_json->>'status' AS status,
|
||||
COUNT(*)::BIGINT AS status_count
|
||||
FROM ${table}
|
||||
GROUP BY state_json->>'status'`,
|
||||
[now, ['leased', 'running']],
|
||||
);
|
||||
const counts = {};
|
||||
let total = 0;
|
||||
let claimable = 0;
|
||||
let expiredLeases = 0;
|
||||
for (const row of result.rows ?? []) {
|
||||
if (row.status) counts[row.status] = Number(row.status_count ?? 0);
|
||||
total += Number(row.status_count ?? 0);
|
||||
claimable += Number(row.claimable ?? 0);
|
||||
expiredLeases += Number(row.expired_leases ?? 0);
|
||||
}
|
||||
return { total, counts, claimable, expiredLeases };
|
||||
},
|
||||
|
||||
async getAdmissionStats({
|
||||
tenantId = null,
|
||||
userId = null,
|
||||
now = Date.now(),
|
||||
windowMs = 60_000,
|
||||
} = {}) {
|
||||
const result = await pool.query(
|
||||
`SELECT
|
||||
COUNT(*) FILTER (
|
||||
WHERE state_json->>'status' = ANY($1::text[])
|
||||
)::BIGINT AS global_active,
|
||||
COUNT(*) FILTER (
|
||||
WHERE state_json->>'status' = ANY($1::text[])
|
||||
AND $2::text IS NOT NULL
|
||||
AND state_json->'request'->'subject'->>'tenantId' = $2
|
||||
)::BIGINT AS tenant_active,
|
||||
COUNT(*) FILTER (
|
||||
WHERE state_json->>'status' = ANY($1::text[])
|
||||
AND $3::text IS NOT NULL
|
||||
AND state_json->'request'->'subject'->>'userId' = $3
|
||||
)::BIGINT AS user_active,
|
||||
COUNT(*) FILTER (
|
||||
WHERE $2::text IS NOT NULL
|
||||
AND state_json->'request'->'subject'->>'tenantId' = $2
|
||||
AND created_at >= $4
|
||||
)::BIGINT AS tenant_recent,
|
||||
COUNT(*) FILTER (
|
||||
WHERE $3::text IS NOT NULL
|
||||
AND state_json->'request'->'subject'->>'userId' = $3
|
||||
AND created_at >= $4
|
||||
)::BIGINT AS user_recent
|
||||
FROM ${table}`,
|
||||
[
|
||||
['queued', 'retryable', 'leased', 'running'],
|
||||
tenantId,
|
||||
userId,
|
||||
now - windowMs,
|
||||
],
|
||||
);
|
||||
const row = result.rows?.[0] ?? {};
|
||||
return {
|
||||
globalActive: Number(row.global_active ?? 0),
|
||||
tenantActive: Number(row.tenant_active ?? 0),
|
||||
userActive: Number(row.user_active ?? 0),
|
||||
tenantRecent: Number(row.tenant_recent ?? 0),
|
||||
userRecent: Number(row.user_recent ?? 0),
|
||||
};
|
||||
},
|
||||
|
||||
async recordWorkerHeartbeat(record) {
|
||||
await pool.query(
|
||||
`INSERT INTO ${workerTable} (worker_id, state_json, last_seen_at)
|
||||
VALUES ($1, $2::jsonb, $3)
|
||||
ON CONFLICT (worker_id) DO UPDATE
|
||||
SET state_json = EXCLUDED.state_json,
|
||||
last_seen_at = EXCLUDED.last_seen_at`,
|
||||
[record.workerId, JSON.stringify(record), record.lastSeenAt],
|
||||
);
|
||||
return structuredClone(record);
|
||||
},
|
||||
|
||||
async listWorkers({ now = Date.now(), staleAfterMs = 60_000 } = {}) {
|
||||
const result = await pool.query(
|
||||
`SELECT state_json
|
||||
FROM ${workerTable}
|
||||
ORDER BY last_seen_at DESC
|
||||
LIMIT 500`,
|
||||
);
|
||||
return (result.rows ?? [])
|
||||
.map(projectRow)
|
||||
.filter(Boolean)
|
||||
.map((worker) => ({
|
||||
...worker,
|
||||
stale: Number(worker.lastSeenAt ?? 0) < now - staleAfterMs,
|
||||
}));
|
||||
},
|
||||
|
||||
async listEvents(jobId, { after = 0, limit = 500 } = {}) {
|
||||
const cursor = Math.max(0, Number(after) || 0);
|
||||
const pageSize = Math.min(500, Math.max(1, Number(limit) || 500));
|
||||
@@ -320,6 +568,8 @@ export const executorJobStoreInternals = {
|
||||
DEFAULT_EXECUTOR_SCHEMA,
|
||||
EXECUTOR_JOB_TABLE,
|
||||
EXECUTOR_JOB_EVENT_TABLE,
|
||||
EXECUTOR_WORKER_TABLE,
|
||||
appendEvent,
|
||||
eventWithSequence,
|
||||
normalizeMode,
|
||||
normalizeSchema,
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
import crypto from 'node:crypto';
|
||||
import {
|
||||
buildExecutorJobEvent,
|
||||
executorGatewayInternals,
|
||||
} from './executor-gateway.mjs';
|
||||
|
||||
const WORKER_PROTOCOL_VERSION = 'executor-worker-v1';
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_MAX_ATTEMPTS = 3;
|
||||
const MAX_PROGRESS_DATA_BYTES = 16 * 1024;
|
||||
const ACTIVE_STATUSES = Object.freeze(['leased', 'running']);
|
||||
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.floor(number)));
|
||||
}
|
||||
|
||||
function protocolError(code, message, status = 409, details = null) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
error.status = status;
|
||||
error.details = details;
|
||||
return error;
|
||||
}
|
||||
|
||||
function requiredIdentifier(value, name, max = 128) {
|
||||
const normalized = String(value ?? '').trim();
|
||||
if (!normalized || !/^[a-zA-Z0-9:_-]+$/.test(normalized) || normalized.length > max) {
|
||||
throw protocolError(
|
||||
`EXECUTOR_${name.toUpperCase()}_INVALID`,
|
||||
`${name} must be a safe identifier no longer than ${max} characters`,
|
||||
422,
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function sanitizeData(value, maxBytes = MAX_PROGRESS_DATA_BYTES) {
|
||||
if (value == null) return null;
|
||||
const encoded = JSON.stringify(value);
|
||||
if (Buffer.byteLength(encoded, 'utf8') > maxBytes) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_EVENT_DATA_TOO_LARGE',
|
||||
`Executor event data exceeds ${maxBytes} bytes`,
|
||||
413,
|
||||
);
|
||||
}
|
||||
return structuredClone(value);
|
||||
}
|
||||
|
||||
function sanitizeError(error) {
|
||||
const source = error && typeof error === 'object' ? error : {};
|
||||
return {
|
||||
code: String(source.code ?? 'EXECUTOR_FAILED').trim().slice(0, 128),
|
||||
message: String(source.message ?? error ?? 'Executor failed').trim().slice(0, 2048),
|
||||
retryable: source.retryable === true,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeResult(result) {
|
||||
const source = result && typeof result === 'object' ? result : {};
|
||||
const artifactRefs = Array.isArray(source.artifactRefs)
|
||||
? source.artifactRefs
|
||||
.map((reference) => {
|
||||
const kind = String(reference?.kind ?? '').trim().slice(0, 64);
|
||||
const id = String(reference?.id ?? '').trim().slice(0, 256);
|
||||
return kind && id ? { kind, id } : null;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.slice(0, 100)
|
||||
: [];
|
||||
return {
|
||||
outcome: String(source.outcome ?? 'completed').trim().slice(0, 128) || 'completed',
|
||||
summary: String(source.summary ?? '').trim().slice(0, 4096) || null,
|
||||
artifactRefs,
|
||||
metrics: source.metrics && typeof source.metrics === 'object'
|
||||
? sanitizeData(source.metrics, 4096)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
function requireWorkerStore(store) {
|
||||
for (const method of [
|
||||
'claimNext',
|
||||
'transition',
|
||||
'listExpiredLeases',
|
||||
'getQueueStats',
|
||||
]) {
|
||||
if (typeof store?.[method] !== 'function') {
|
||||
throw new Error(`Executor Worker store missing method: ${method}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function requireApplied(result, action) {
|
||||
if (result?.applied) return result.record;
|
||||
const code = result?.reason === 'not_found'
|
||||
? 'EXECUTOR_JOB_NOT_FOUND'
|
||||
: result?.reason === 'lease_mismatch'
|
||||
? 'EXECUTOR_LEASE_LOST'
|
||||
: 'EXECUTOR_JOB_STATE_CONFLICT';
|
||||
const status = result?.reason === 'not_found' ? 404 : 409;
|
||||
throw protocolError(code, `Cannot ${action} executor job: ${result?.reason ?? 'conflict'}`, status, {
|
||||
currentStatus: result?.record?.status ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export function createExecutorWorkerCoordinator({
|
||||
store,
|
||||
nowMs = Date.now,
|
||||
randomUUID = crypto.randomUUID,
|
||||
defaultLeaseDurationMs = DEFAULT_LEASE_DURATION_MS,
|
||||
defaultMaxAttempts = DEFAULT_MAX_ATTEMPTS,
|
||||
workerStaleAfterMs = 60_000,
|
||||
} = {}) {
|
||||
requireWorkerStore(store);
|
||||
|
||||
async function claim({
|
||||
workerId,
|
||||
executors,
|
||||
leaseDurationMs = defaultLeaseDurationMs,
|
||||
draining = false,
|
||||
} = {}) {
|
||||
const normalizedWorkerId = requiredIdentifier(workerId, 'worker_id');
|
||||
const normalizedExecutors = [...new Set(
|
||||
(Array.isArray(executors) ? executors : [executors])
|
||||
.map((executor) => String(executor ?? '').trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
)].slice(0, 20);
|
||||
if (!normalizedExecutors.length) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_WORKER_EXECUTORS_REQUIRED',
|
||||
'Worker claim requires at least one executor',
|
||||
422,
|
||||
);
|
||||
}
|
||||
const now = nowMs();
|
||||
await store.recordWorkerHeartbeat?.({
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
workerId: normalizedWorkerId,
|
||||
executors: normalizedExecutors,
|
||||
draining: draining === true,
|
||||
lastSeenAt: now,
|
||||
});
|
||||
if (draining === true) return null;
|
||||
const duration = clampInteger(leaseDurationMs, defaultLeaseDurationMs, 5_000, 5 * 60_000);
|
||||
const leaseToken = randomUUID();
|
||||
const claimed = await store.claimNext({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: normalizedExecutors,
|
||||
now,
|
||||
leaseToken,
|
||||
leaseDurationMs: duration,
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_leased',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: normalizedWorkerId,
|
||||
executor: updated.executor,
|
||||
attempt: updated.attempts,
|
||||
leaseExpiresAt: updated.lease.expiresAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return claimed
|
||||
? {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
job: claimed,
|
||||
leaseToken,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
async function start(jobId, { leaseToken } = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const now = nowMs();
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: ['leased'],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
status: 'running',
|
||||
startedAt: current.startedAt ?? now,
|
||||
updatedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_started',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: updated.lease.workerId,
|
||||
attempt: updated.attempts,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'start');
|
||||
}
|
||||
|
||||
async function heartbeat(jobId, {
|
||||
leaseToken,
|
||||
leaseDurationMs = defaultLeaseDurationMs,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const now = nowMs();
|
||||
const duration = clampInteger(leaseDurationMs, defaultLeaseDurationMs, 5_000, 5 * 60_000);
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
lease: {
|
||||
...current.lease,
|
||||
heartbeatAt: now,
|
||||
expiresAt: now + duration,
|
||||
},
|
||||
updatedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_heartbeat',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: updated.lease.workerId,
|
||||
leaseExpiresAt: updated.lease.expiresAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'heartbeat');
|
||||
}
|
||||
|
||||
async function progress(jobId, {
|
||||
leaseToken,
|
||||
type = 'executor_job_progress',
|
||||
data = null,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const normalizedType = String(type ?? '').trim();
|
||||
if (!/^executor_job_[a-z0-9_]{1,96}$/.test(normalizedType)) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_PROGRESS_TYPE_INVALID',
|
||||
'Progress event type must use the executor_job_ namespace',
|
||||
422,
|
||||
);
|
||||
}
|
||||
const safeData = sanitizeData(data);
|
||||
const now = nowMs();
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return { ...current, updatedAt: now };
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: normalizedType,
|
||||
timestamp: now,
|
||||
data: safeData,
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'append progress');
|
||||
}
|
||||
|
||||
async function complete(jobId, { leaseToken, result: output = null } = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const safeResult = sanitizeResult(output);
|
||||
const now = nowMs();
|
||||
const transition = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
status: 'succeeded',
|
||||
reason: null,
|
||||
result: safeResult,
|
||||
lease: null,
|
||||
updatedAt: now,
|
||||
completedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_succeeded',
|
||||
timestamp: now,
|
||||
data: safeResult,
|
||||
}),
|
||||
});
|
||||
return requireApplied(transition, 'complete');
|
||||
}
|
||||
|
||||
async function fail(jobId, {
|
||||
leaseToken,
|
||||
error,
|
||||
retryDelayMs = 1_000,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const safeError = sanitizeError(error);
|
||||
const now = nowMs();
|
||||
const transition = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
const maxAttempts = clampInteger(
|
||||
current.maxAttempts,
|
||||
defaultMaxAttempts,
|
||||
1,
|
||||
20,
|
||||
);
|
||||
const retryable = safeError.retryable && Number(current.attempts ?? 0) < maxAttempts;
|
||||
return {
|
||||
...current,
|
||||
status: retryable ? 'retryable' : 'failed',
|
||||
reason: safeError.code,
|
||||
error: safeError,
|
||||
lease: null,
|
||||
nextAttemptAt: retryable
|
||||
? now + clampInteger(retryDelayMs, 1_000, 0, 60 * 60_000)
|
||||
: null,
|
||||
updatedAt: now,
|
||||
completedAt: retryable ? null : now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: updated.status === 'retryable'
|
||||
? 'executor_job_retry_scheduled'
|
||||
: 'executor_job_failed',
|
||||
timestamp: now,
|
||||
data: {
|
||||
status: updated.status,
|
||||
error: safeError,
|
||||
nextAttemptAt: updated.nextAttemptAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(transition, 'fail');
|
||||
}
|
||||
|
||||
async function recoverExpired({ limit = 100 } = {}) {
|
||||
const now = nowMs();
|
||||
const expired = await store.listExpiredLeases({
|
||||
now,
|
||||
limit: clampInteger(limit, 100, 1, 500),
|
||||
});
|
||||
const recovered = [];
|
||||
for (const candidate of expired) {
|
||||
const leaseToken = candidate.lease?.token;
|
||||
if (!leaseToken) continue;
|
||||
const transition = await store.transition(candidate.jobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken,
|
||||
updater(current) {
|
||||
const maxAttempts = clampInteger(
|
||||
current.maxAttempts,
|
||||
defaultMaxAttempts,
|
||||
1,
|
||||
20,
|
||||
);
|
||||
const retryable = Number(current.attempts ?? 0) < maxAttempts;
|
||||
return {
|
||||
...current,
|
||||
status: retryable ? 'retryable' : 'timed_out',
|
||||
reason: 'executor_lease_expired',
|
||||
error: {
|
||||
code: 'EXECUTOR_LEASE_EXPIRED',
|
||||
message: 'Worker lease expired before completion',
|
||||
retryable,
|
||||
},
|
||||
lease: null,
|
||||
nextAttemptAt: retryable ? now : null,
|
||||
updatedAt: now,
|
||||
completedAt: retryable ? null : now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: updated.status === 'retryable'
|
||||
? 'executor_job_lease_recovered'
|
||||
: 'executor_job_timed_out',
|
||||
timestamp: now,
|
||||
data: {
|
||||
previousWorkerId: candidate.lease?.workerId ?? null,
|
||||
status: updated.status,
|
||||
attempt: updated.attempts,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (transition.applied) recovered.push(transition.record);
|
||||
}
|
||||
return {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
inspected: expired.length,
|
||||
recovered,
|
||||
};
|
||||
}
|
||||
|
||||
async function stats() {
|
||||
const now = nowMs();
|
||||
const workers = typeof store.listWorkers === 'function'
|
||||
? await store.listWorkers({ now, staleAfterMs: workerStaleAfterMs })
|
||||
: [];
|
||||
return {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
...(await store.getQueueStats({ now })),
|
||||
workers,
|
||||
healthyWorkers: workers.filter((worker) => !worker.stale && !worker.draining).length,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
claim,
|
||||
start,
|
||||
heartbeat,
|
||||
progress,
|
||||
complete,
|
||||
fail,
|
||||
recoverExpired,
|
||||
stats,
|
||||
};
|
||||
}
|
||||
|
||||
export const executorWorkerProtocolInternals = {
|
||||
ACTIVE_STATUSES,
|
||||
DEFAULT_LEASE_DURATION_MS,
|
||||
DEFAULT_MAX_ATTEMPTS,
|
||||
MAX_PROGRESS_DATA_BYTES,
|
||||
WORKER_PROTOCOL_VERSION,
|
||||
clampInteger,
|
||||
protocolError,
|
||||
requiredIdentifier,
|
||||
sanitizeData,
|
||||
sanitizeError,
|
||||
sanitizeResult,
|
||||
terminalStatuses: executorGatewayInternals.TERMINAL_JOB_STATUSES,
|
||||
};
|
||||
@@ -0,0 +1,181 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import { createInMemoryExecutorJobStore } from './executor-gateway.mjs';
|
||||
import { createExecutorWorkerCoordinator } from './executor-worker-protocol.mjs';
|
||||
|
||||
function queuedJob(overrides = {}) {
|
||||
return {
|
||||
version: 'executor-job-state-v1',
|
||||
jobId: 'job-1',
|
||||
idempotencyKey: 'idem-1',
|
||||
requestFingerprint: 'a'.repeat(64),
|
||||
executor: 'goosed',
|
||||
status: 'queued',
|
||||
attempts: 0,
|
||||
maxAttempts: 2,
|
||||
createdAt: 1000,
|
||||
updatedAt: 1000,
|
||||
completedAt: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function seed(store, job = queuedJob()) {
|
||||
await store.createIfAbsent(job);
|
||||
}
|
||||
|
||||
test('worker claims, starts, heartbeats and completes a queued job with lease fencing', async () => {
|
||||
let now = 1000;
|
||||
let token = 0;
|
||||
const store = createInMemoryExecutorJobStore();
|
||||
await seed(store);
|
||||
const coordinator = createExecutorWorkerCoordinator({
|
||||
store,
|
||||
nowMs: () => now,
|
||||
randomUUID: () => `lease-${++token}`,
|
||||
});
|
||||
|
||||
const claimed = await coordinator.claim({
|
||||
workerId: 'worker-1',
|
||||
executors: ['goosed'],
|
||||
leaseDurationMs: 5000,
|
||||
});
|
||||
assert.equal(claimed.version, 'executor-worker-v1');
|
||||
assert.equal(claimed.leaseToken, 'lease-1');
|
||||
assert.equal(claimed.job.status, 'leased');
|
||||
assert.equal(claimed.job.attempts, 1);
|
||||
assert.equal(await coordinator.claim({
|
||||
workerId: 'worker-2',
|
||||
executors: ['goosed'],
|
||||
}), null);
|
||||
|
||||
await assert.rejects(
|
||||
coordinator.start('job-1', { leaseToken: 'wrong-token' }),
|
||||
(error) => error.code === 'EXECUTOR_LEASE_LOST',
|
||||
);
|
||||
const started = await coordinator.start('job-1', { leaseToken: 'lease-1' });
|
||||
assert.equal(started.status, 'running');
|
||||
assert.equal(started.startedAt, 1000);
|
||||
|
||||
now = 3000;
|
||||
const heartbeat = await coordinator.heartbeat('job-1', {
|
||||
leaseToken: 'lease-1',
|
||||
leaseDurationMs: 5000,
|
||||
});
|
||||
assert.equal(heartbeat.lease.heartbeatAt, 3000);
|
||||
assert.equal(heartbeat.lease.expiresAt, 8000);
|
||||
|
||||
await coordinator.progress('job-1', {
|
||||
leaseToken: 'lease-1',
|
||||
type: 'executor_job_output_chunk',
|
||||
data: { characters: 12 },
|
||||
});
|
||||
now = 4000;
|
||||
const completed = await coordinator.complete('job-1', {
|
||||
leaseToken: 'lease-1',
|
||||
result: {
|
||||
summary: 'done',
|
||||
artifactRefs: [{ kind: 'patch', id: 'artifact-1' }],
|
||||
},
|
||||
});
|
||||
assert.equal(completed.status, 'succeeded');
|
||||
assert.equal(completed.lease, null);
|
||||
assert.deepEqual(completed.result.artifactRefs, [{ kind: 'patch', id: 'artifact-1' }]);
|
||||
assert.deepEqual(
|
||||
(await store.listEvents('job-1')).map((event) => event.type),
|
||||
[
|
||||
'executor_job_leased',
|
||||
'executor_job_started',
|
||||
'executor_job_heartbeat',
|
||||
'executor_job_output_chunk',
|
||||
'executor_job_succeeded',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('retryable failure is delayed and a later terminal failure consumes max attempts', async () => {
|
||||
let now = 1000;
|
||||
let token = 0;
|
||||
const store = createInMemoryExecutorJobStore();
|
||||
await seed(store);
|
||||
const coordinator = createExecutorWorkerCoordinator({
|
||||
store,
|
||||
nowMs: () => now,
|
||||
randomUUID: () => `lease-${++token}`,
|
||||
});
|
||||
|
||||
const first = await coordinator.claim({ workerId: 'worker-1', executors: ['goosed'] });
|
||||
await coordinator.start('job-1', { leaseToken: first.leaseToken });
|
||||
const retryable = await coordinator.fail('job-1', {
|
||||
leaseToken: first.leaseToken,
|
||||
error: { code: 'UPSTREAM_BUSY', message: 'busy', retryable: true },
|
||||
retryDelayMs: 5000,
|
||||
});
|
||||
assert.equal(retryable.status, 'retryable');
|
||||
assert.equal(retryable.nextAttemptAt, 6000);
|
||||
assert.equal(await coordinator.claim({
|
||||
workerId: 'worker-2',
|
||||
executors: ['goosed'],
|
||||
}), null);
|
||||
|
||||
now = 6000;
|
||||
const second = await coordinator.claim({ workerId: 'worker-2', executors: ['goosed'] });
|
||||
assert.equal(second.job.attempts, 2);
|
||||
await coordinator.start('job-1', { leaseToken: second.leaseToken });
|
||||
const failed = await coordinator.fail('job-1', {
|
||||
leaseToken: second.leaseToken,
|
||||
error: { code: 'UPSTREAM_BUSY', message: 'still busy', retryable: true },
|
||||
});
|
||||
assert.equal(failed.status, 'failed');
|
||||
assert.equal(failed.completedAt, 6000);
|
||||
});
|
||||
|
||||
test('expired leases are recovered with fencing and eventually time out', async () => {
|
||||
let now = 1000;
|
||||
let token = 0;
|
||||
const store = createInMemoryExecutorJobStore();
|
||||
await seed(store, queuedJob({ maxAttempts: 1 }));
|
||||
const coordinator = createExecutorWorkerCoordinator({
|
||||
store,
|
||||
nowMs: () => now,
|
||||
randomUUID: () => `lease-${++token}`,
|
||||
});
|
||||
const claimed = await coordinator.claim({
|
||||
workerId: 'worker-crashed',
|
||||
executors: ['goosed'],
|
||||
leaseDurationMs: 5000,
|
||||
});
|
||||
await coordinator.start('job-1', { leaseToken: claimed.leaseToken });
|
||||
now = 6001;
|
||||
const recovery = await coordinator.recoverExpired();
|
||||
assert.equal(recovery.inspected, 1);
|
||||
assert.equal(recovery.recovered[0].status, 'timed_out');
|
||||
await assert.rejects(
|
||||
coordinator.heartbeat('job-1', { leaseToken: claimed.leaseToken }),
|
||||
(error) => error.code === 'EXECUTOR_JOB_STATE_CONFLICT',
|
||||
);
|
||||
const stats = await coordinator.stats();
|
||||
assert.equal(stats.counts.timed_out, 1);
|
||||
assert.equal(stats.expiredLeases, 0);
|
||||
});
|
||||
|
||||
test('worker protocol bounds identifiers and progress payloads', async () => {
|
||||
const store = createInMemoryExecutorJobStore();
|
||||
await seed(store);
|
||||
const coordinator = createExecutorWorkerCoordinator({
|
||||
store,
|
||||
randomUUID: () => 'lease-1',
|
||||
});
|
||||
await assert.rejects(
|
||||
coordinator.claim({ workerId: '../worker', executors: ['goosed'] }),
|
||||
(error) => error.code === 'EXECUTOR_WORKER_ID_INVALID',
|
||||
);
|
||||
const claimed = await coordinator.claim({ workerId: 'worker-1', executors: ['goosed'] });
|
||||
await assert.rejects(
|
||||
coordinator.progress('job-1', {
|
||||
leaseToken: claimed.leaseToken,
|
||||
data: { output: 'x'.repeat(20_000) },
|
||||
}),
|
||||
(error) => error.code === 'EXECUTOR_EVENT_DATA_TOO_LARGE',
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
function workerError(code, message, options = {}) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
error.retryable = options.retryable === true;
|
||||
return error;
|
||||
}
|
||||
|
||||
function safeJson(response) {
|
||||
return response.json().catch(() => null);
|
||||
}
|
||||
|
||||
export function createExecutorWorkerClient({
|
||||
baseUrl,
|
||||
serviceToken = '',
|
||||
workerToken = '',
|
||||
fetchImpl = globalThis.fetch,
|
||||
} = {}) {
|
||||
const target = String(baseUrl ?? '').trim().replace(/\/$/, '');
|
||||
if (!target) throw new Error('Executor worker client requires baseUrl');
|
||||
if (typeof fetchImpl !== 'function') throw new Error('Executor worker client requires fetch');
|
||||
|
||||
async function request(path, { method = 'GET', body = null } = {}) {
|
||||
const response = await fetchImpl(`${target}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
...(body == null ? {} : { 'content-type': 'application/json' }),
|
||||
...(serviceToken ? { authorization: `Bearer ${serviceToken}` } : {}),
|
||||
...(workerToken ? { 'x-orchestrator-worker-token': workerToken } : {}),
|
||||
},
|
||||
...(body == null ? {} : { body: JSON.stringify(body) }),
|
||||
});
|
||||
if (response.status === 204) return null;
|
||||
const payload = await safeJson(response);
|
||||
if (!response.ok) {
|
||||
const error = workerError(
|
||||
payload?.error?.code ?? 'EXECUTOR_WORKER_HTTP_ERROR',
|
||||
payload?.error?.message ?? `Executor worker request failed (${response.status})`,
|
||||
{ retryable: response.status >= 500 || response.status === 429 },
|
||||
);
|
||||
error.status = response.status;
|
||||
throw error;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
return {
|
||||
claim(input) {
|
||||
return request('/v1/workers/claim', { method: 'POST', body: input });
|
||||
},
|
||||
start(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/start`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
heartbeat(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/heartbeat`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
progress(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/progress`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
complete(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/complete`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
fail(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/fail`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
recoverExpired(input = {}) {
|
||||
return request('/v1/workers/recover-expired', { method: 'POST', body: input });
|
||||
},
|
||||
stats() {
|
||||
return request('/v1/workers/stats');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerAdapterRegistry(initialAdapters = []) {
|
||||
const adapters = new Map();
|
||||
return {
|
||||
register(adapter) {
|
||||
const id = String(adapter?.id ?? '').trim().toLowerCase();
|
||||
if (!id || typeof adapter?.submit !== 'function') {
|
||||
throw new Error('Worker adapter requires id and submit');
|
||||
}
|
||||
if (adapters.has(id)) throw new Error(`Worker adapter already registered: ${id}`);
|
||||
adapters.set(id, adapter);
|
||||
return adapter;
|
||||
},
|
||||
get(id) {
|
||||
return adapters.get(String(id ?? '').trim().toLowerCase()) ?? null;
|
||||
},
|
||||
list() {
|
||||
return [...adapters.values()].map((adapter) => ({
|
||||
id: adapter.id,
|
||||
kind: adapter.kind ?? 'worker',
|
||||
capabilities: [...(adapter.capabilities ?? [])],
|
||||
}));
|
||||
},
|
||||
ids() {
|
||||
return [...adapters.keys()];
|
||||
},
|
||||
_initialize() {
|
||||
for (const adapter of initialAdapters) this.register(adapter);
|
||||
return this;
|
||||
},
|
||||
}._initialize();
|
||||
}
|
||||
|
||||
export function createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry = createWorkerAdapterRegistry(),
|
||||
workerId,
|
||||
heartbeatMs = 10_000,
|
||||
leaseDurationMs = 30_000,
|
||||
logger = console,
|
||||
} = {}) {
|
||||
const normalizedWorkerId = String(workerId ?? '').trim();
|
||||
if (!normalizedWorkerId) throw new Error('Executor worker runtime requires workerId');
|
||||
if (!client?.claim) throw new Error('Executor worker runtime requires client');
|
||||
let stopping = false;
|
||||
const active = new Map();
|
||||
|
||||
async function executeAdapter(adapter, job, leaseToken, controller) {
|
||||
return adapter.submit(job.request, {
|
||||
jobId: job.jobId,
|
||||
signal: controller.signal,
|
||||
emit(type, data) {
|
||||
return client.progress(job.jobId, { leaseToken, type, data });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function processClaim(claim) {
|
||||
const job = claim.job;
|
||||
const leaseToken = claim.leaseToken;
|
||||
const controller = new AbortController();
|
||||
active.set(job.jobId, { controller, leaseToken });
|
||||
const timeoutMs = Math.max(500, Number(job.timeoutMs ?? job.request?.controls?.timeoutMs) || 900_000);
|
||||
const timeout = setTimeout(() => controller.abort('executor_timeout'), timeoutMs);
|
||||
let heartbeatFailure = null;
|
||||
const heartbeat = setInterval(() => {
|
||||
void client.heartbeat(job.jobId, { leaseToken, leaseDurationMs }).catch((error) => {
|
||||
heartbeatFailure = error;
|
||||
controller.abort('heartbeat_failed');
|
||||
});
|
||||
}, Math.max(1_000, Math.min(heartbeatMs, Math.floor(leaseDurationMs / 2))));
|
||||
heartbeat.unref?.();
|
||||
try {
|
||||
await client.start(job.jobId, { leaseToken });
|
||||
if (!job.request) {
|
||||
throw workerError(
|
||||
'EXECUTOR_JOB_REQUEST_MISSING',
|
||||
'Claimed executor job does not contain an execution request',
|
||||
);
|
||||
}
|
||||
const primary = registry.get(job.executor);
|
||||
if (!primary) {
|
||||
throw workerError(
|
||||
'EXECUTOR_ADAPTER_NOT_INSTALLED',
|
||||
`Worker does not have adapter ${job.executor}`,
|
||||
);
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
result = await executeAdapter(primary, job, leaseToken, controller);
|
||||
} catch (primaryError) {
|
||||
const fallbackId = String(job.fallbackExecutor ?? '').trim();
|
||||
const fallback = job.fallbackAvailable ? registry.get(fallbackId) : null;
|
||||
if (!fallback || controller.signal.aborted) throw primaryError;
|
||||
await client.progress(job.jobId, {
|
||||
leaseToken,
|
||||
type: 'executor_job_fallback_started',
|
||||
data: {
|
||||
from: job.executor,
|
||||
to: fallbackId,
|
||||
reason: String(primaryError?.code ?? 'EXECUTOR_PRIMARY_FAILED').slice(0, 128),
|
||||
},
|
||||
});
|
||||
result = await executeAdapter(fallback, {
|
||||
...job,
|
||||
executor: fallbackId,
|
||||
request: { ...job.request, executor: fallbackId },
|
||||
}, leaseToken, controller);
|
||||
}
|
||||
if (heartbeatFailure) throw heartbeatFailure;
|
||||
await client.complete(job.jobId, { leaseToken, result });
|
||||
return { processed: true, jobId: job.jobId, status: 'succeeded' };
|
||||
} catch (error) {
|
||||
const failure = controller.signal.aborted && !heartbeatFailure
|
||||
? {
|
||||
code: 'EXECUTOR_TIMEOUT_OR_CANCELLED',
|
||||
message: 'Executor was cancelled or exceeded its timeout',
|
||||
retryable: false,
|
||||
}
|
||||
: {
|
||||
code: String(error?.code ?? 'EXECUTOR_WORKER_FAILED').slice(0, 128),
|
||||
message: String(error?.message ?? error ?? 'Executor worker failed').slice(0, 2048),
|
||||
retryable: error?.retryable === true,
|
||||
};
|
||||
try {
|
||||
await client.fail(job.jobId, {
|
||||
leaseToken,
|
||||
error: failure,
|
||||
retryDelayMs: failure.retryable ? 2_000 : 0,
|
||||
});
|
||||
} catch (reportError) {
|
||||
logger.error?.(
|
||||
`[executor-worker] failed to report ${job.jobId}: ${reportError.message}`,
|
||||
);
|
||||
throw reportError;
|
||||
}
|
||||
return { processed: true, jobId: job.jobId, status: 'failed', error: failure };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
clearInterval(heartbeat);
|
||||
active.delete(job.jobId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async runOnce() {
|
||||
if (stopping) return { processed: false, reason: 'stopping' };
|
||||
const claim = await client.claim({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: registry.ids(),
|
||||
leaseDurationMs,
|
||||
});
|
||||
if (!claim) return { processed: false, reason: 'queue_empty' };
|
||||
return processClaim(claim);
|
||||
},
|
||||
async recoverExpired(options = {}) {
|
||||
return client.recoverExpired(options);
|
||||
},
|
||||
status() {
|
||||
return {
|
||||
version: 'executor-worker-runtime-v1',
|
||||
workerId: normalizedWorkerId,
|
||||
stopping,
|
||||
activeJobs: [...active.keys()],
|
||||
adapters: registry.list(),
|
||||
heartbeatMs,
|
||||
leaseDurationMs,
|
||||
};
|
||||
},
|
||||
async stop() {
|
||||
stopping = true;
|
||||
await client.claim({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: registry.ids(),
|
||||
leaseDurationMs,
|
||||
draining: true,
|
||||
}).catch(() => null);
|
||||
for (const { controller } of active.values()) controller.abort('worker_stopping');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const executorWorkerRuntimeInternals = {
|
||||
safeJson,
|
||||
workerError,
|
||||
};
|
||||
@@ -0,0 +1,143 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
createExecutorWorkerRuntime,
|
||||
createWorkerAdapterRegistry,
|
||||
} from './executor-worker-runtime.mjs';
|
||||
|
||||
function claim(overrides = {}) {
|
||||
return {
|
||||
leaseToken: 'lease-1',
|
||||
job: {
|
||||
jobId: 'job-1',
|
||||
executor: 'goosed',
|
||||
fallbackExecutor: 'aider',
|
||||
fallbackAvailable: true,
|
||||
timeoutMs: 30_000,
|
||||
request: {
|
||||
executor: 'goosed',
|
||||
task: {
|
||||
instruction: 'do it',
|
||||
workspaceRef: { kind: 'workspace-alias', id: 'canary' },
|
||||
},
|
||||
},
|
||||
...overrides,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeClient(nextClaim = claim()) {
|
||||
const calls = [];
|
||||
let claimed = false;
|
||||
return {
|
||||
calls,
|
||||
async claim(input) {
|
||||
calls.push(['claim', input]);
|
||||
if (claimed) return null;
|
||||
claimed = true;
|
||||
return nextClaim;
|
||||
},
|
||||
async start(jobId, input) {
|
||||
calls.push(['start', jobId, input]);
|
||||
},
|
||||
async heartbeat(jobId, input) {
|
||||
calls.push(['heartbeat', jobId, input]);
|
||||
},
|
||||
async progress(jobId, input) {
|
||||
calls.push(['progress', jobId, input]);
|
||||
},
|
||||
async complete(jobId, input) {
|
||||
calls.push(['complete', jobId, input]);
|
||||
},
|
||||
async fail(jobId, input) {
|
||||
calls.push(['fail', jobId, input]);
|
||||
},
|
||||
async recoverExpired(input) {
|
||||
calls.push(['recover', input]);
|
||||
return { recovered: [] };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function adapter(id, submit) {
|
||||
return { id, capabilities: [], submit };
|
||||
}
|
||||
|
||||
test('worker runtime claims, executes, emits progress and completes through lease protocol', async () => {
|
||||
const client = fakeClient();
|
||||
const registry = createWorkerAdapterRegistry([
|
||||
adapter('goosed', async (_request, { emit }) => {
|
||||
await emit('executor_job_adapter_event', { eventType: 'finish' });
|
||||
return {
|
||||
outcome: 'completed',
|
||||
artifactRefs: [{ kind: 'goosed-session', id: 'session-1' }],
|
||||
};
|
||||
}),
|
||||
]);
|
||||
const worker = createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry,
|
||||
workerId: 'worker-1',
|
||||
});
|
||||
const result = await worker.runOnce();
|
||||
assert.equal(result.status, 'succeeded');
|
||||
assert.deepEqual(
|
||||
client.calls.map((call) => call[0]),
|
||||
['claim', 'start', 'progress', 'complete'],
|
||||
);
|
||||
assert.equal(client.calls.at(-1)[2].leaseToken, 'lease-1');
|
||||
assert.deepEqual(worker.status().activeJobs, []);
|
||||
});
|
||||
|
||||
test('worker runtime falls back to an installed adapter and records the transition', async () => {
|
||||
const client = fakeClient();
|
||||
const registry = createWorkerAdapterRegistry([
|
||||
adapter('goosed', async () => {
|
||||
const error = new Error('primary failed');
|
||||
error.code = 'PRIMARY_FAILED';
|
||||
throw error;
|
||||
}),
|
||||
adapter('aider', async (request) => ({
|
||||
outcome: 'completed',
|
||||
summary: request.executor,
|
||||
artifactRefs: [{ kind: 'patch', id: 'patch-1' }],
|
||||
})),
|
||||
]);
|
||||
const worker = createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry,
|
||||
workerId: 'worker-1',
|
||||
});
|
||||
const result = await worker.runOnce();
|
||||
assert.equal(result.status, 'succeeded');
|
||||
const fallback = client.calls.find(
|
||||
(call) => call[0] === 'progress' && call[2].type === 'executor_job_fallback_started',
|
||||
);
|
||||
assert.deepEqual(fallback[2].data, {
|
||||
from: 'goosed',
|
||||
to: 'aider',
|
||||
reason: 'PRIMARY_FAILED',
|
||||
});
|
||||
assert.equal(client.calls.at(-1)[0], 'complete');
|
||||
});
|
||||
|
||||
test('worker runtime reports bounded adapter failures without completing the job', async () => {
|
||||
const client = fakeClient(claim({ fallbackAvailable: false }));
|
||||
const registry = createWorkerAdapterRegistry([
|
||||
adapter('goosed', async () => {
|
||||
const error = new Error('upstream unavailable');
|
||||
error.code = 'UPSTREAM_UNAVAILABLE';
|
||||
error.retryable = true;
|
||||
throw error;
|
||||
}),
|
||||
]);
|
||||
const worker = createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry,
|
||||
workerId: 'worker-1',
|
||||
});
|
||||
const result = await worker.runOnce();
|
||||
assert.equal(result.status, 'failed');
|
||||
assert.equal(client.calls.at(-1)[0], 'fail');
|
||||
assert.equal(client.calls.at(-1)[2].error.retryable, true);
|
||||
});
|
||||
@@ -1,9 +1,15 @@
|
||||
import { normalizeRunSpec } from './contracts.mjs';
|
||||
import { buildRunEvent, normalizeRunSpec } from './contracts.mjs';
|
||||
import {
|
||||
CODE_RUN_SHADOW_WORKFLOW,
|
||||
createCodeRunShadowGraph,
|
||||
} from './code-run-shadow-graph.mjs';
|
||||
import { createExecutorGateway } from './executor-gateway.mjs';
|
||||
import {
|
||||
createExecutorGateway,
|
||||
createInMemoryExecutorJobStore,
|
||||
createPhase5ExecutorAdapterRegistry,
|
||||
} from './executor-gateway.mjs';
|
||||
import { createExecutorWorkerCoordinator } from './executor-worker-protocol.mjs';
|
||||
import { createExecutorAdmissionPolicy } from './executor-admission-policy.mjs';
|
||||
|
||||
function graphConfig(runId) {
|
||||
return {
|
||||
@@ -38,11 +44,32 @@ export function createLangGraphOrchestratorRuntime({
|
||||
checkpointProbe = null,
|
||||
executorJobStore = null,
|
||||
executorJobProbe = null,
|
||||
executionEnabled = false,
|
||||
enabledExecutors = [],
|
||||
workerOptions = {},
|
||||
admissionConfig = {},
|
||||
} = {}) {
|
||||
const executorGateway = createExecutorGateway({
|
||||
...(executorJobStore ? { store: executorJobStore } : {}),
|
||||
const jobStore = executorJobStore ?? createInMemoryExecutorJobStore();
|
||||
const admissionPolicy = createExecutorAdmissionPolicy({
|
||||
store: jobStore,
|
||||
config: admissionConfig,
|
||||
nowMs: workerOptions.nowMs,
|
||||
});
|
||||
const executorGateway = createExecutorGateway({
|
||||
store: jobStore,
|
||||
registry: createPhase5ExecutorAdapterRegistry({ enabledExecutors }),
|
||||
executionEnabled,
|
||||
admissionPolicy,
|
||||
});
|
||||
const workerCoordinator = createExecutorWorkerCoordinator({
|
||||
store: jobStore,
|
||||
...workerOptions,
|
||||
});
|
||||
const graph = createCodeRunShadowGraph({
|
||||
checkpointer,
|
||||
executorGateway,
|
||||
executionEnabled,
|
||||
});
|
||||
const graph = createCodeRunShadowGraph({ checkpointer, executorGateway });
|
||||
const probe = typeof checkpointProbe === 'function'
|
||||
? checkpointProbe
|
||||
: async () => {
|
||||
@@ -57,10 +84,54 @@ export function createLangGraphOrchestratorRuntime({
|
||||
return graph.getState(graphConfig(runId));
|
||||
}
|
||||
|
||||
async function refreshExecutorTerminal(runId, snapshot) {
|
||||
const values = snapshot?.values ?? {};
|
||||
if (values.status !== 'waiting') return snapshot;
|
||||
const jobId = values.plan?.executorJob?.id;
|
||||
if (!jobId) return snapshot;
|
||||
const job = await executorGateway.getJob(jobId);
|
||||
const terminal = {
|
||||
succeeded: { status: 'succeeded', phase: 'completed' },
|
||||
failed: { status: 'failed', phase: 'executor_failed' },
|
||||
cancelled: { status: 'cancelled', phase: 'cancelled' },
|
||||
timed_out: { status: 'failed', phase: 'executor_timed_out' },
|
||||
}[job?.status];
|
||||
if (!terminal) return snapshot;
|
||||
const result = {
|
||||
observed: false,
|
||||
executed: job.status === 'succeeded',
|
||||
executorAdapter: values.plan.executorAdapter,
|
||||
taskType: values.plan.taskType,
|
||||
executorJob: {
|
||||
id: job.jobId,
|
||||
status: job.status,
|
||||
reason: job.reason,
|
||||
result: job.result ?? null,
|
||||
},
|
||||
};
|
||||
await graph.updateState(graphConfig(runId), {
|
||||
...terminal,
|
||||
result,
|
||||
events: [buildRunEvent({
|
||||
runId,
|
||||
sequence: (values.events?.length ?? 0) + 1,
|
||||
type: job.status === 'succeeded'
|
||||
? 'workflow_completed'
|
||||
: `workflow_${job.status}`,
|
||||
data: result,
|
||||
})],
|
||||
});
|
||||
return getSnapshot(runId);
|
||||
}
|
||||
|
||||
async function getState(runId) {
|
||||
const normalizedRunId = String(runId ?? '').trim();
|
||||
if (!normalizedRunId) return null;
|
||||
return projectSnapshot(normalizedRunId, await getSnapshot(normalizedRunId));
|
||||
const snapshot = await getSnapshot(normalizedRunId);
|
||||
return projectSnapshot(
|
||||
normalizedRunId,
|
||||
await refreshExecutorTerminal(normalizedRunId, snapshot),
|
||||
);
|
||||
}
|
||||
|
||||
function health() {
|
||||
@@ -71,8 +142,13 @@ export function createLangGraphOrchestratorRuntime({
|
||||
kind: checkpointKind,
|
||||
durable: Boolean(durable),
|
||||
},
|
||||
execution: 'observe-only',
|
||||
execution: executionEnabled ? 'worker-queue-enabled' : 'observe-only',
|
||||
executorGateway: executorGateway.status(),
|
||||
workerProtocol: {
|
||||
version: 'executor-worker-v1',
|
||||
enabled: executionEnabled === true,
|
||||
},
|
||||
admissionPolicy: admissionPolicy.status(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -100,7 +176,10 @@ export function createLangGraphOrchestratorRuntime({
|
||||
|
||||
async listEvents(runId, { after = 0 } = {}) {
|
||||
const normalizedRunId = String(runId ?? '').trim();
|
||||
const snapshot = normalizedRunId ? await getSnapshot(normalizedRunId) : null;
|
||||
const initial = normalizedRunId ? await getSnapshot(normalizedRunId) : null;
|
||||
const snapshot = normalizedRunId
|
||||
? await refreshExecutorTerminal(normalizedRunId, initial)
|
||||
: null;
|
||||
if (!snapshot?.values?.spec) return null;
|
||||
const cursor = Math.max(0, Number(after) || 0);
|
||||
const events = (snapshot.values.events ?? []).filter((event) => event.sequence > cursor);
|
||||
@@ -114,6 +193,7 @@ export function createLangGraphOrchestratorRuntime({
|
||||
async resume(runId) {
|
||||
const state = await getState(runId);
|
||||
if (!state) return null;
|
||||
if (state.status === 'waiting') return state;
|
||||
const error = new Error('This workflow has no interrupt point to resume');
|
||||
error.code = 'WORKFLOW_NOT_INTERRUPTED';
|
||||
error.status = 409;
|
||||
@@ -123,6 +203,12 @@ export function createLangGraphOrchestratorRuntime({
|
||||
async cancel(runId) {
|
||||
const state = await getState(runId);
|
||||
if (!state) return null;
|
||||
if (state.status === 'waiting' && state.plan?.executorJob?.id) {
|
||||
await executorGateway.cancel(state.plan.executorJob.id, {
|
||||
reason: 'workflow_cancelled',
|
||||
});
|
||||
return getState(runId);
|
||||
}
|
||||
const error = new Error('Completed shadow observations cannot be cancelled');
|
||||
error.code = 'WORKFLOW_ALREADY_TERMINAL';
|
||||
error.status = 409;
|
||||
@@ -137,13 +223,69 @@ export function createLangGraphOrchestratorRuntime({
|
||||
return executorGateway.listEvents(jobId, options);
|
||||
},
|
||||
|
||||
submitExecutorJob(input) {
|
||||
return executorGateway.createJob(input);
|
||||
},
|
||||
|
||||
cancelExecutorJob(jobId, options = {}) {
|
||||
return executorGateway.cancel(jobId, options);
|
||||
},
|
||||
|
||||
claimExecutorJob(input) {
|
||||
return workerCoordinator.claim(input);
|
||||
},
|
||||
|
||||
startExecutorJob(jobId, input) {
|
||||
return workerCoordinator.start(jobId, input);
|
||||
},
|
||||
|
||||
heartbeatExecutorJob(jobId, input) {
|
||||
return workerCoordinator.heartbeat(jobId, input);
|
||||
},
|
||||
|
||||
progressExecutorJob(jobId, input) {
|
||||
return workerCoordinator.progress(jobId, input);
|
||||
},
|
||||
|
||||
completeExecutorJob(jobId, input) {
|
||||
return workerCoordinator.complete(jobId, input);
|
||||
},
|
||||
|
||||
failExecutorJob(jobId, input) {
|
||||
return workerCoordinator.fail(jobId, input);
|
||||
},
|
||||
|
||||
recoverExpiredExecutorJobs(input = {}) {
|
||||
return workerCoordinator.recoverExpired(input);
|
||||
},
|
||||
|
||||
getExecutorQueueStats() {
|
||||
return workerCoordinator.stats();
|
||||
},
|
||||
|
||||
async metrics() {
|
||||
return {
|
||||
version: 'orchestrator-metrics-v1',
|
||||
timestamp: Date.now(),
|
||||
queue: await workerCoordinator.stats(),
|
||||
gateway: executorGateway.status(),
|
||||
};
|
||||
},
|
||||
|
||||
health,
|
||||
|
||||
async ready() {
|
||||
await Promise.all([probe(), probeExecutorJobs()]);
|
||||
const executorQueue = await workerCoordinator.stats();
|
||||
if (executionEnabled && executorQueue.healthyWorkers < 1) {
|
||||
const error = new Error('Execution is enabled but no healthy executor worker is registered');
|
||||
error.code = 'EXECUTOR_WORKER_UNAVAILABLE';
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
...health(),
|
||||
ready: true,
|
||||
executorQueue,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
@@ -41,7 +41,7 @@ test('LangGraph runtime checkpoints a deterministic observe-only code workflow',
|
||||
id: 'run-shadow-1:executor-preview',
|
||||
executor: 'goosed',
|
||||
status: 'blocked',
|
||||
reason: 'executor_dispatch_not_implemented',
|
||||
reason: 'executor_execution_disabled',
|
||||
durable: false,
|
||||
});
|
||||
assert.equal(
|
||||
@@ -153,3 +153,60 @@ test('LangGraph runtime derives a bounded opaque Executor Job id for long run id
|
||||
assert.ok(state.plan.executorJob.id.length <= 128);
|
||||
assert.equal((await runtime.getExecutorJob(state.plan.executorJob.id)).status, 'blocked');
|
||||
});
|
||||
|
||||
test('LangGraph active workflow waits on a leased Executor Job and converges terminal state', async () => {
|
||||
const runtime = createLangGraphOrchestratorRuntime({
|
||||
checkpointer: new MemorySaver(),
|
||||
executionEnabled: true,
|
||||
enabledExecutors: ['goosed'],
|
||||
});
|
||||
const state = await runtime.start({
|
||||
runId: 'run-active-1',
|
||||
requestId: 'request-active-1',
|
||||
workflow: { name: 'code-run-v1', version: 1 },
|
||||
subject: { tenantId: 'tenant-1', userId: 'user-1' },
|
||||
input: {
|
||||
instruction: 'Run a controlled canary',
|
||||
executor: 'goosed',
|
||||
workspaceRef: { kind: 'workspace-alias', id: 'canary' },
|
||||
},
|
||||
policy: {
|
||||
executionMode: 'active',
|
||||
sideEffectsAllowed: true,
|
||||
networkAllowed: false,
|
||||
},
|
||||
});
|
||||
assert.equal(state.status, 'waiting');
|
||||
assert.equal(state.phase, 'executor_queued');
|
||||
assert.equal(state.plan.executorJob.status, 'queued');
|
||||
|
||||
const claim = await runtime.claimExecutorJob({
|
||||
workerId: 'worker-1',
|
||||
executors: ['goosed'],
|
||||
});
|
||||
await runtime.startExecutorJob(claim.job.jobId, { leaseToken: claim.leaseToken });
|
||||
await runtime.completeExecutorJob(claim.job.jobId, {
|
||||
leaseToken: claim.leaseToken,
|
||||
result: {
|
||||
summary: 'canary complete',
|
||||
artifactRefs: [{ kind: 'canary-result', id: 'result-1' }],
|
||||
},
|
||||
});
|
||||
|
||||
const completed = await runtime.getState('run-active-1');
|
||||
assert.equal(completed.status, 'succeeded');
|
||||
assert.equal(completed.result.executed, true);
|
||||
assert.deepEqual(
|
||||
completed.result.executorJob.result.artifactRefs,
|
||||
[{ kind: 'canary-result', id: 'result-1' }],
|
||||
);
|
||||
assert.deepEqual(
|
||||
(await runtime.listEvents('run-active-1')).events.map((event) => event.type),
|
||||
[
|
||||
'workflow_validated',
|
||||
'workflow_planned',
|
||||
'workflow_executor_queued',
|
||||
'workflow_completed',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
@@ -10,6 +10,21 @@ function positivePort(value, fallback = 8093) {
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function envFlag(value, fallback = false) {
|
||||
const normalized = String(value ?? '').trim().toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
return ['1', 'true', 'yes', 'on'].includes(normalized);
|
||||
}
|
||||
|
||||
function csvList(value) {
|
||||
return [...new Set(
|
||||
String(value ?? '')
|
||||
.split(',')
|
||||
.map((item) => item.trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
)];
|
||||
}
|
||||
|
||||
export async function startOrchestratorServer({
|
||||
env = process.env,
|
||||
logger = console,
|
||||
@@ -37,10 +52,25 @@ export async function startOrchestratorServer({
|
||||
checkpointProbe: checkpoint.probe,
|
||||
executorJobStore: executorJobs.store,
|
||||
executorJobProbe: executorJobs.probe,
|
||||
executionEnabled: envFlag(env.MEMIND_ORCHESTRATOR_EXECUTION_ENABLED, false),
|
||||
enabledExecutors: csvList(env.MEMIND_ORCHESTRATOR_ENABLED_EXECUTORS),
|
||||
admissionConfig: {
|
||||
tenantAllowlist: csvList(env.MEMIND_ORCHESTRATOR_TENANT_ALLOWLIST),
|
||||
userAllowlist: csvList(env.MEMIND_ORCHESTRATOR_USER_ALLOWLIST),
|
||||
workspaceKindAllowlist: csvList(
|
||||
env.MEMIND_ORCHESTRATOR_WORKSPACE_KIND_ALLOWLIST ?? 'workspace-alias',
|
||||
),
|
||||
maxGlobalActive: env.MEMIND_ORCHESTRATOR_MAX_GLOBAL_ACTIVE,
|
||||
maxTenantActive: env.MEMIND_ORCHESTRATOR_MAX_TENANT_ACTIVE,
|
||||
maxUserActive: env.MEMIND_ORCHESTRATOR_MAX_USER_ACTIVE,
|
||||
maxTenantPerMinute: env.MEMIND_ORCHESTRATOR_MAX_TENANT_PER_MINUTE,
|
||||
maxUserPerMinute: env.MEMIND_ORCHESTRATOR_MAX_USER_PER_MINUTE,
|
||||
},
|
||||
});
|
||||
const app = createOrchestratorApp({
|
||||
runtime,
|
||||
serviceToken: env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
||||
workerToken: env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
|
||||
});
|
||||
const host = String(env.MEMIND_ORCHESTRATOR_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
|
||||
const port = positivePort(env.MEMIND_ORCHESTRATOR_PORT, 8093);
|
||||
@@ -99,5 +129,7 @@ if (isEntrypoint) {
|
||||
}
|
||||
|
||||
export const orchestratorServerInternals = {
|
||||
csvList,
|
||||
envFlag,
|
||||
positivePort,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
import os from 'node:os';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import {
|
||||
createAiderExecutorAdapter,
|
||||
createGoosedExecutorAdapter,
|
||||
createOpenHandsExecutorAdapter,
|
||||
} from './executor-adapters.mjs';
|
||||
import {
|
||||
createExecutorWorkerClient,
|
||||
createExecutorWorkerRuntime,
|
||||
createWorkerAdapterRegistry,
|
||||
} from './executor-worker-runtime.mjs';
|
||||
|
||||
function envFlag(value, fallback = false) {
|
||||
const normalized = String(value ?? '').trim().toLowerCase();
|
||||
if (!normalized) return fallback;
|
||||
return ['1', 'true', 'yes', 'on'].includes(normalized);
|
||||
}
|
||||
|
||||
function positiveInteger(value, fallback) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.floor(number) : fallback;
|
||||
}
|
||||
|
||||
function jsonObject(value, fallback = {}) {
|
||||
try {
|
||||
const parsed = JSON.parse(String(value ?? ''));
|
||||
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : fallback;
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
export function createConfiguredWorker({ env = process.env, fetchImpl = globalThis.fetch } = {}) {
|
||||
const registry = createWorkerAdapterRegistry();
|
||||
const workspaceAliases = jsonObject(env.MEMIND_EXECUTOR_WORKSPACE_ALIASES_JSON);
|
||||
const allowedRoot = String(env.MEMIND_EXECUTOR_WORKSPACE_ROOT ?? '').trim();
|
||||
const goosedUrl = String(env.MEMIND_EXECUTOR_GOOSED_URL ?? '').trim();
|
||||
if (goosedUrl) {
|
||||
registry.register(createGoosedExecutorAdapter({
|
||||
baseUrl: goosedUrl,
|
||||
secret: env.MEMIND_EXECUTOR_GOOSED_TOKEN,
|
||||
fetchImpl,
|
||||
allowRemote: envFlag(env.MEMIND_EXECUTOR_ALLOW_REMOTE_GOOSED, false),
|
||||
workspaceResolver: async (reference) => workspaceAliases[reference?.id] ?? null,
|
||||
}));
|
||||
}
|
||||
if (env.MEMIND_EXECUTOR_AIDER_COMMAND && allowedRoot) {
|
||||
registry.register(createAiderExecutorAdapter({
|
||||
command: env.MEMIND_EXECUTOR_AIDER_COMMAND,
|
||||
allowedRoot,
|
||||
workspaceAliases,
|
||||
}));
|
||||
}
|
||||
if (env.MEMIND_EXECUTOR_OPENHANDS_COMMAND && allowedRoot) {
|
||||
registry.register(createOpenHandsExecutorAdapter({
|
||||
command: env.MEMIND_EXECUTOR_OPENHANDS_COMMAND,
|
||||
allowedRoot,
|
||||
workspaceAliases,
|
||||
}));
|
||||
}
|
||||
if (!registry.ids().length) {
|
||||
throw new Error('Executor worker has no configured adapters');
|
||||
}
|
||||
const client = createExecutorWorkerClient({
|
||||
baseUrl: env.MEMIND_ORCHESTRATOR_URL ?? 'http://127.0.0.1:8093',
|
||||
serviceToken: env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
||||
workerToken: env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
|
||||
fetchImpl,
|
||||
});
|
||||
return createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry,
|
||||
workerId: env.MEMIND_EXECUTOR_WORKER_ID ?? `${os.hostname()}-${process.pid}`,
|
||||
heartbeatMs: positiveInteger(env.MEMIND_EXECUTOR_HEARTBEAT_MS, 10_000),
|
||||
leaseDurationMs: positiveInteger(env.MEMIND_EXECUTOR_LEASE_MS, 30_000),
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const worker = createConfiguredWorker();
|
||||
const once = process.argv.includes('--once');
|
||||
const recover = process.argv.includes('--recover-expired');
|
||||
if (recover) {
|
||||
console.log(JSON.stringify(await worker.recoverExpired(), null, 2));
|
||||
return;
|
||||
}
|
||||
if (once) {
|
||||
console.log(JSON.stringify(await worker.runOnce(), null, 2));
|
||||
return;
|
||||
}
|
||||
let stopping = false;
|
||||
const stop = async () => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
await worker.stop();
|
||||
};
|
||||
process.once('SIGINT', () => void stop());
|
||||
process.once('SIGTERM', () => void stop());
|
||||
const pollMs = positiveInteger(process.env.MEMIND_EXECUTOR_POLL_MS, 1000);
|
||||
console.log(JSON.stringify({ ok: true, ...worker.status() }));
|
||||
while (!stopping) {
|
||||
await worker.runOnce();
|
||||
await new Promise((resolve) => setTimeout(resolve, pollMs));
|
||||
}
|
||||
}
|
||||
|
||||
const isEntrypoint = process.argv[1]
|
||||
&& import.meta.url === pathToFileURL(process.argv[1]).href;
|
||||
if (isEntrypoint) {
|
||||
await main();
|
||||
}
|
||||
|
||||
export const executorWorkerEntrypointInternals = {
|
||||
envFlag,
|
||||
jsonObject,
|
||||
positiveInteger,
|
||||
};
|
||||
Reference in New Issue
Block a user