tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: harden orchestrator execution runtime

Browse Source
This commit is contained in:
john
2026-07-24 23:53:32 +08:00
parent 396bb78200
commit f6f2cd0933
55 changed files with 5122 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/orchestrator/executor-gateway.mjs
+298 -26
View File
@@ -4,7 +4,7 @@ import { normalizeWorkflowEngineId } from './contracts.mjs';
const EXECUTOR_JOB_REQUEST_VERSION = 'executor-job-request-v1';
const EXECUTOR_JOB_STATE_VERSION = 'executor-job-state-v1';
const EXECUTOR_JOB_EVENT_VERSION = 'executor-job-event-v1';
const EXECUTOR_DISPATCH_IMPLEMENTED = false;
const EXECUTOR_DISPATCH_IMPLEMENTED = true;
const MAX_INSTRUCTION_CHARACTERS = 32_000;
const MAX_EVENT_PAGE_SIZE = 500;
const TERMINAL_JOB_STATUSES = new Set([
@@ -167,6 +167,7 @@ export function normalizeExecutorJobRequest(input = {}) {
timeoutMs: clampInteger(source.controls?.timeoutMs, 15 * 60_000, 500, 60 * 60_000),
cancellationAllowed: source.controls?.cancellationAllowed !== false,
fallbackExecutor: normalizeWorkflowEngineId(source.controls?.fallbackExecutor, ''),
maxAttempts: clampInteger(source.controls?.maxAttempts, 3, 1, 20),
},
metadata: source.metadata && typeof source.metadata === 'object'
? clone(source.metadata)
@@ -252,6 +253,34 @@ export function createDisabledExecutorAdapter(descriptor) {
};
}
export function createWorkerQueueExecutorAdapter(descriptor, { enabled = false } = {}) {
const id = normalizeWorkflowEngineId(descriptor?.id, '');
if (!id) throw new Error('Worker queue executor adapter requires a valid id');
const reject = () => {
throw gatewayError(
'EXECUTOR_WORKER_CLAIM_REQUIRED',
`Executor adapter ${id} is dispatched through the worker claim protocol`,
);
};
return {
id,
label: descriptor.label ?? id,
kind: descriptor.kind ?? 'worker',
enabled,
dispatchImplemented: true,
status: enabled ? 'worker-queue' : 'disabled',
capabilities: Array.isArray(descriptor.capabilities)
? [...descriptor.capabilities, 'lease-fencing']
: ['lease-fencing'],
submit: reject,
cancel: reject,
getState: reject,
async *streamEvents() {
reject();
},
};
}
export function createPhase3ExecutorAdapterRegistry() {
return createExecutorAdapterRegistry(
PHASE3_EXECUTOR_DESCRIPTORS.map(createDisabledExecutorAdapter),
@@ -262,10 +291,49 @@ export function listPhase3ExecutorAdapters() {
return createPhase3ExecutorAdapterRegistry().list();
}
export function createPhase5ExecutorAdapterRegistry({ enabledExecutors = [] } = {}) {
const enabled = new Set(
(Array.isArray(enabledExecutors) ? enabledExecutors : [])
.map((id) => normalizeWorkflowEngineId(id, ''))
.filter(Boolean),
);
return createExecutorAdapterRegistry(
PHASE3_EXECUTOR_DESCRIPTORS.map((descriptor) => (
createWorkerQueueExecutorAdapter(descriptor, { enabled: enabled.has(descriptor.id) })
)),
);
}
export function listPhase5ExecutorAdapters(options = {}) {
return createPhase5ExecutorAdapterRegistry(options).list();
}
export function projectExecutorJobForRead(record) {
if (!record) return null;
const {
request: _request,
requestFingerprint: _requestFingerprint,
lease,
...safe
} = clone(record);
return {
...safe,
lease: lease
? {
workerId: lease.workerId ?? null,
acquiredAt: lease.acquiredAt ?? null,
heartbeatAt: lease.heartbeatAt ?? null,
expiresAt: lease.expiresAt ?? null,
}
: null,
};
}
export function createInMemoryExecutorJobStore() {
const byId = new Map();
const byIdempotencyKey = new Map();
const eventsByJobId = new Map();
const workersById = new Map();
function appendEvent(jobId, event) {
if (!event) return;
@@ -278,6 +346,14 @@ export function createInMemoryExecutorJobStore() {
eventsByJobId.set(jobId, events);
}
function transitionResult(applied, record, reason = null) {
return {
applied,
record: record ? clone(record) : null,
reason,
};
}
return {
kind: 'memory',
durable: false,
@@ -307,6 +383,160 @@ export function createInMemoryExecutorJobStore() {
appendEvent(record.jobId, event);
return clone(record);
},
async claimNext({
workerId,
executors,
now,
leaseToken,
leaseDurationMs,
eventFactory,
}) {
const allowedExecutors = new Set(executors);
const candidate = [...byId.values()]
.filter((record) => (
allowedExecutors.has(record.executor)
&& (
record.status === 'queued'
|| (
record.status === 'retryable'
&& Number(record.nextAttemptAt ?? 0) <= now
)
)
))
.sort((left, right) => (
Number(left.createdAt ?? 0) - Number(right.createdAt ?? 0)
|| String(left.jobId).localeCompare(String(right.jobId))
))[0];
if (!candidate) return null;
const updated = {
...clone(candidate),
status: 'leased',
reason: null,
attempts: Number(candidate.attempts ?? 0) + 1,
nextAttemptAt: null,
lease: {
token: leaseToken,
workerId,
acquiredAt: now,
heartbeatAt: now,
expiresAt: now + leaseDurationMs,
},
updatedAt: now,
completedAt: null,
};
byId.set(updated.jobId, clone(updated));
appendEvent(
updated.jobId,
typeof eventFactory === 'function' ? eventFactory(updated, candidate) : null,
);
return clone(updated);
},
async transition(jobId, {
expectedStatuses = [],
leaseToken = null,
updater,
eventFactory = null,
} = {}) {
const normalizedJobId = String(jobId ?? '').trim();
const current = byId.get(normalizedJobId);
if (!current) return transitionResult(false, null, 'not_found');
if (expectedStatuses.length && !expectedStatuses.includes(current.status)) {
return transitionResult(false, current, 'status_mismatch');
}
if (leaseToken != null && current.lease?.token !== leaseToken) {
return transitionResult(false, current, 'lease_mismatch');
}
const updated = updater(clone(current));
if (!updated || updated.jobId !== current.jobId) {
throw new Error('Executor job transition must preserve job identity');
}
byId.set(normalizedJobId, clone(updated));
appendEvent(
normalizedJobId,
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
);
return transitionResult(true, updated);
},
async listExpiredLeases({ now, limit = 100 } = {}) {
return [...byId.values()]
.filter((record) => (
['leased', 'running'].includes(record.status)
&& Number(record.lease?.expiresAt ?? 0) <= now
))
.sort((left, right) => (
Number(left.lease?.expiresAt ?? 0) - Number(right.lease?.expiresAt ?? 0)
))
.slice(0, clampInteger(limit, 100, 1, 500))
.map(clone);
},
async getQueueStats({ now = Date.now() } = {}) {
const records = [...byId.values()];
const counts = {};
for (const record of records) {
counts[record.status] = Number(counts[record.status] ?? 0) + 1;
}
return {
total: records.length,
counts,
claimable: records.filter((record) => (
record.status === 'queued'
|| (
record.status === 'retryable'
&& Number(record.nextAttemptAt ?? 0) <= now
)
)).length,
expiredLeases: records.filter((record) => (
['leased', 'running'].includes(record.status)
&& Number(record.lease?.expiresAt ?? 0) <= now
)).length,
};
},
async getAdmissionStats({
tenantId = null,
userId = null,
now = Date.now(),
windowMs = 60_000,
} = {}) {
const activeStatuses = new Set(['queued', 'retryable', 'leased', 'running']);
const records = [...byId.values()];
const matchesTenant = (record) => (
tenantId != null && record.request?.subject?.tenantId === tenantId
);
const matchesUser = (record) => (
userId != null && record.request?.subject?.userId === userId
);
return {
globalActive: records.filter((record) => activeStatuses.has(record.status)).length,
tenantActive: tenantId == null
? 0
: records.filter((record) => activeStatuses.has(record.status) && matchesTenant(record)).length,
userActive: userId == null
? 0
: records.filter((record) => activeStatuses.has(record.status) && matchesUser(record)).length,
tenantRecent: tenantId == null
? 0
: records.filter((record) => (
matchesTenant(record) && Number(record.createdAt ?? 0) >= now - windowMs
)).length,
userRecent: userId == null
? 0
: records.filter((record) => (
matchesUser(record) && Number(record.createdAt ?? 0) >= now - windowMs
)).length,
};
},
async recordWorkerHeartbeat(record) {
workersById.set(record.workerId, clone(record));
return clone(record);
},
async listWorkers({ now = Date.now(), staleAfterMs = 60_000 } = {}) {
return [...workersById.values()]
.sort((left, right) => Number(right.lastSeenAt) - Number(left.lastSeenAt))
.map((worker) => ({
...clone(worker),
stale: Number(worker.lastSeenAt ?? 0) < now - staleAfterMs,
}));
},
async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
const cursor = Math.max(0, Number(after) || 0);
const pageSize = clampInteger(limit, MAX_EVENT_PAGE_SIZE, 1, MAX_EVENT_PAGE_SIZE);
@@ -336,6 +566,8 @@ export function createExecutorGateway({
registry = createPhase3ExecutorAdapterRegistry(),
store = createInMemoryExecutorJobStore(),
nowMs = Date.now,
executionEnabled = false,
admissionPolicy = null,
} = {}) {
if (!registry?.has || !registry?.list) {
throw new Error('Executor Gateway requires an adapter registry');
@@ -352,6 +584,7 @@ export function createExecutorGateway({
&& fallbackAdapter?.dispatchImplemented === true;
const gates = {
implementation: EXECUTOR_DISPATCH_IMPLEMENTED,
serviceEnabled: executionEnabled === true,
adapterRegistered: Boolean(adapter),
adapterEnabled: adapter?.enabled === true,
executionAuthorized: request.authorization.executionAllowed,
@@ -367,9 +600,21 @@ export function createExecutorGateway({
fallbackRegistered,
fallbackAvailable,
dispatchAllowed: Object.values(gates).every(Boolean),
reason: EXECUTOR_DISPATCH_IMPLEMENTED
? 'executor_adapter_unavailable'
: 'executor_dispatch_not_implemented',
reason: !EXECUTOR_DISPATCH_IMPLEMENTED
? 'executor_dispatch_not_implemented'
: executionEnabled !== true
? 'executor_execution_disabled'
: !adapter
? 'executor_adapter_unregistered'
: adapter.enabled !== true
? 'executor_adapter_disabled'
: !request.authorization.executionAllowed
? 'executor_execution_unauthorized'
: !request.policy.sideEffectsAllowed
? 'executor_side_effects_not_allowed'
: !request.task.workspaceRef
? 'executor_workspace_required'
: 'queued_for_worker',
gates,
};
}
@@ -396,40 +641,56 @@ export function createExecutorGateway({
}
return { created: false, job: existing };
}
const decision = previewNormalized(request);
let decision = previewNormalized(request);
if (decision.dispatchAllowed && admissionPolicy?.evaluate) {
const admission = await admissionPolicy.evaluate(request);
if (!admission.allowed) {
decision = {
...decision,
dispatchAllowed: false,
reason: admission.reason,
admission,
};
} else {
decision = { ...decision, admission };
}
}
const now = nowMs();
const dispatchAllowed = decision.dispatchAllowed;
const record = {
version: EXECUTOR_JOB_STATE_VERSION,
jobId: request.jobId,
idempotencyKey: request.idempotencyKey,
requestFingerprint,
executor: request.executor,
status: 'blocked',
status: dispatchAllowed ? 'queued' : 'blocked',
reason: decision.reason,
attempts: 0,
dispatchAllowed: false,
maxAttempts: request.controls.maxAttempts,
dispatchAllowed,
cancellationAllowed: request.controls.cancellationAllowed,
timeoutMs: request.controls.timeoutMs,
fallbackExecutor: decision.fallbackExecutor,
fallbackRegistered: decision.fallbackRegistered,
fallbackAvailable: decision.fallbackAvailable,
decision,
request: dispatchAllowed ? request : null,
createdAt: now,
updatedAt: now,
startedAt: null,
completedAt: now,
completedAt: dispatchAllowed ? null : now,
};
const result = await store.createIfAbsent(record, {
initialEvent: buildExecutorJobEvent({
jobId: request.jobId,
sequence: 1,
type: 'executor_job_blocked',
type: dispatchAllowed ? 'executor_job_queued' : 'executor_job_blocked',
timestamp: now,
data: {
executor: request.executor,
status: record.status,
reason: record.reason,
dispatchAllowed: false,
dispatchAllowed,
},
}),
});
@@ -451,32 +712,42 @@ export function createExecutorGateway({
if (current.status === 'cancelled' || !current.cancellationAllowed) return current;
if (TERMINAL_JOB_STATUSES.has(current.status) && current.status !== 'blocked') return current;
const now = nowMs();
const updated = {
...current,
const applyCancellation = (record) => ({
...record,
status: 'cancelled',
reason: String(reason).slice(0, 256),
lease: null,
updatedAt: now,
completedAt: now,
};
return store.update(updated, {
event: buildExecutorJobEvent({
jobId: current.jobId,
type: 'executor_job_cancelled',
timestamp: now,
data: {
status: updated.status,
reason: updated.reason,
},
}),
});
const eventForCancellation = (updated) => buildExecutorJobEvent({
jobId: updated.jobId,
type: 'executor_job_cancelled',
timestamp: now,
data: {
status: updated.status,
reason: updated.reason,
},
});
if (typeof store.transition === 'function') {
const result = await store.transition(current.jobId, {
expectedStatuses: ['blocked', 'queued', 'retryable', 'leased', 'running'],
updater: applyCancellation,
eventFactory: eventForCancellation,
});
return result.applied ? result.record : result.record;
}
const updated = applyCancellation(current);
return store.update(updated, { event: eventForCancellation(updated) });
}
return {
preview,
createJob,
cancel,
getJob(jobId) {
return store.getById(jobId);
async getJob(jobId, { includeRequest = false } = {}) {
const record = await store.getById(jobId);
return includeRequest ? record : projectExecutorJobForRead(record);
},
async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
const normalizedJobId = String(jobId ?? '').trim();
@@ -501,7 +772,7 @@ export function createExecutorGateway({
return {
version: 'executor-gateway-status-v1',
dispatchImplemented: EXECUTOR_DISPATCH_IMPLEMENTED,
executionEnabled: false,
executionEnabled: executionEnabled === true,
store: {
kind: String(store.kind ?? (store.durable ? 'durable' : 'memory')),
durable: store.durable === true,
@@ -522,5 +793,6 @@ export const executorGatewayInternals = {
PHASE3_EXECUTOR_DESCRIPTORS,
TERMINAL_JOB_STATUSES,
fingerprint,
projectExecutorJobForRead,
stableValue,
};