feat: harden orchestrator execution runtime
This commit is contained in:
john
@@ -0,0 +1,446 @@
|
||||
import crypto from 'node:crypto';
|
||||
import {
|
||||
buildExecutorJobEvent,
|
||||
executorGatewayInternals,
|
||||
} from './executor-gateway.mjs';
|
||||
|
||||
const WORKER_PROTOCOL_VERSION = 'executor-worker-v1';
|
||||
const DEFAULT_LEASE_DURATION_MS = 30_000;
|
||||
const DEFAULT_MAX_ATTEMPTS = 3;
|
||||
const MAX_PROGRESS_DATA_BYTES = 16 * 1024;
|
||||
const ACTIVE_STATUSES = Object.freeze(['leased', 'running']);
|
||||
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.floor(number)));
|
||||
}
|
||||
|
||||
function protocolError(code, message, status = 409, details = null) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
error.status = status;
|
||||
error.details = details;
|
||||
return error;
|
||||
}
|
||||
|
||||
function requiredIdentifier(value, name, max = 128) {
|
||||
const normalized = String(value ?? '').trim();
|
||||
if (!normalized || !/^[a-zA-Z0-9:_-]+$/.test(normalized) || normalized.length > max) {
|
||||
throw protocolError(
|
||||
`EXECUTOR_${name.toUpperCase()}_INVALID`,
|
||||
`${name} must be a safe identifier no longer than ${max} characters`,
|
||||
422,
|
||||
);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function sanitizeData(value, maxBytes = MAX_PROGRESS_DATA_BYTES) {
|
||||
if (value == null) return null;
|
||||
const encoded = JSON.stringify(value);
|
||||
if (Buffer.byteLength(encoded, 'utf8') > maxBytes) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_EVENT_DATA_TOO_LARGE',
|
||||
`Executor event data exceeds ${maxBytes} bytes`,
|
||||
413,
|
||||
);
|
||||
}
|
||||
return structuredClone(value);
|
||||
}
|
||||
|
||||
function sanitizeError(error) {
|
||||
const source = error && typeof error === 'object' ? error : {};
|
||||
return {
|
||||
code: String(source.code ?? 'EXECUTOR_FAILED').trim().slice(0, 128),
|
||||
message: String(source.message ?? error ?? 'Executor failed').trim().slice(0, 2048),
|
||||
retryable: source.retryable === true,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeResult(result) {
|
||||
const source = result && typeof result === 'object' ? result : {};
|
||||
const artifactRefs = Array.isArray(source.artifactRefs)
|
||||
? source.artifactRefs
|
||||
.map((reference) => {
|
||||
const kind = String(reference?.kind ?? '').trim().slice(0, 64);
|
||||
const id = String(reference?.id ?? '').trim().slice(0, 256);
|
||||
return kind && id ? { kind, id } : null;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.slice(0, 100)
|
||||
: [];
|
||||
return {
|
||||
outcome: String(source.outcome ?? 'completed').trim().slice(0, 128) || 'completed',
|
||||
summary: String(source.summary ?? '').trim().slice(0, 4096) || null,
|
||||
artifactRefs,
|
||||
metrics: source.metrics && typeof source.metrics === 'object'
|
||||
? sanitizeData(source.metrics, 4096)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
function requireWorkerStore(store) {
|
||||
for (const method of [
|
||||
'claimNext',
|
||||
'transition',
|
||||
'listExpiredLeases',
|
||||
'getQueueStats',
|
||||
]) {
|
||||
if (typeof store?.[method] !== 'function') {
|
||||
throw new Error(`Executor Worker store missing method: ${method}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function requireApplied(result, action) {
|
||||
if (result?.applied) return result.record;
|
||||
const code = result?.reason === 'not_found'
|
||||
? 'EXECUTOR_JOB_NOT_FOUND'
|
||||
: result?.reason === 'lease_mismatch'
|
||||
? 'EXECUTOR_LEASE_LOST'
|
||||
: 'EXECUTOR_JOB_STATE_CONFLICT';
|
||||
const status = result?.reason === 'not_found' ? 404 : 409;
|
||||
throw protocolError(code, `Cannot ${action} executor job: ${result?.reason ?? 'conflict'}`, status, {
|
||||
currentStatus: result?.record?.status ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
export function createExecutorWorkerCoordinator({
|
||||
store,
|
||||
nowMs = Date.now,
|
||||
randomUUID = crypto.randomUUID,
|
||||
defaultLeaseDurationMs = DEFAULT_LEASE_DURATION_MS,
|
||||
defaultMaxAttempts = DEFAULT_MAX_ATTEMPTS,
|
||||
workerStaleAfterMs = 60_000,
|
||||
} = {}) {
|
||||
requireWorkerStore(store);
|
||||
|
||||
async function claim({
|
||||
workerId,
|
||||
executors,
|
||||
leaseDurationMs = defaultLeaseDurationMs,
|
||||
draining = false,
|
||||
} = {}) {
|
||||
const normalizedWorkerId = requiredIdentifier(workerId, 'worker_id');
|
||||
const normalizedExecutors = [...new Set(
|
||||
(Array.isArray(executors) ? executors : [executors])
|
||||
.map((executor) => String(executor ?? '').trim().toLowerCase())
|
||||
.filter(Boolean),
|
||||
)].slice(0, 20);
|
||||
if (!normalizedExecutors.length) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_WORKER_EXECUTORS_REQUIRED',
|
||||
'Worker claim requires at least one executor',
|
||||
422,
|
||||
);
|
||||
}
|
||||
const now = nowMs();
|
||||
await store.recordWorkerHeartbeat?.({
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
workerId: normalizedWorkerId,
|
||||
executors: normalizedExecutors,
|
||||
draining: draining === true,
|
||||
lastSeenAt: now,
|
||||
});
|
||||
if (draining === true) return null;
|
||||
const duration = clampInteger(leaseDurationMs, defaultLeaseDurationMs, 5_000, 5 * 60_000);
|
||||
const leaseToken = randomUUID();
|
||||
const claimed = await store.claimNext({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: normalizedExecutors,
|
||||
now,
|
||||
leaseToken,
|
||||
leaseDurationMs: duration,
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_leased',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: normalizedWorkerId,
|
||||
executor: updated.executor,
|
||||
attempt: updated.attempts,
|
||||
leaseExpiresAt: updated.lease.expiresAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return claimed
|
||||
? {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
job: claimed,
|
||||
leaseToken,
|
||||
}
|
||||
: null;
|
||||
}
|
||||
|
||||
async function start(jobId, { leaseToken } = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const now = nowMs();
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: ['leased'],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
status: 'running',
|
||||
startedAt: current.startedAt ?? now,
|
||||
updatedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_started',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: updated.lease.workerId,
|
||||
attempt: updated.attempts,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'start');
|
||||
}
|
||||
|
||||
async function heartbeat(jobId, {
|
||||
leaseToken,
|
||||
leaseDurationMs = defaultLeaseDurationMs,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const now = nowMs();
|
||||
const duration = clampInteger(leaseDurationMs, defaultLeaseDurationMs, 5_000, 5 * 60_000);
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
lease: {
|
||||
...current.lease,
|
||||
heartbeatAt: now,
|
||||
expiresAt: now + duration,
|
||||
},
|
||||
updatedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_heartbeat',
|
||||
timestamp: now,
|
||||
data: {
|
||||
workerId: updated.lease.workerId,
|
||||
leaseExpiresAt: updated.lease.expiresAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'heartbeat');
|
||||
}
|
||||
|
||||
async function progress(jobId, {
|
||||
leaseToken,
|
||||
type = 'executor_job_progress',
|
||||
data = null,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const normalizedType = String(type ?? '').trim();
|
||||
if (!/^executor_job_[a-z0-9_]{1,96}$/.test(normalizedType)) {
|
||||
throw protocolError(
|
||||
'EXECUTOR_PROGRESS_TYPE_INVALID',
|
||||
'Progress event type must use the executor_job_ namespace',
|
||||
422,
|
||||
);
|
||||
}
|
||||
const safeData = sanitizeData(data);
|
||||
const now = nowMs();
|
||||
const result = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return { ...current, updatedAt: now };
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: normalizedType,
|
||||
timestamp: now,
|
||||
data: safeData,
|
||||
}),
|
||||
});
|
||||
return requireApplied(result, 'append progress');
|
||||
}
|
||||
|
||||
async function complete(jobId, { leaseToken, result: output = null } = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const safeResult = sanitizeResult(output);
|
||||
const now = nowMs();
|
||||
const transition = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
return {
|
||||
...current,
|
||||
status: 'succeeded',
|
||||
reason: null,
|
||||
result: safeResult,
|
||||
lease: null,
|
||||
updatedAt: now,
|
||||
completedAt: now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: 'executor_job_succeeded',
|
||||
timestamp: now,
|
||||
data: safeResult,
|
||||
}),
|
||||
});
|
||||
return requireApplied(transition, 'complete');
|
||||
}
|
||||
|
||||
async function fail(jobId, {
|
||||
leaseToken,
|
||||
error,
|
||||
retryDelayMs = 1_000,
|
||||
} = {}) {
|
||||
const normalizedJobId = requiredIdentifier(jobId, 'job_id');
|
||||
const normalizedLeaseToken = requiredIdentifier(leaseToken, 'lease_token');
|
||||
const safeError = sanitizeError(error);
|
||||
const now = nowMs();
|
||||
const transition = await store.transition(normalizedJobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken: normalizedLeaseToken,
|
||||
updater(current) {
|
||||
const maxAttempts = clampInteger(
|
||||
current.maxAttempts,
|
||||
defaultMaxAttempts,
|
||||
1,
|
||||
20,
|
||||
);
|
||||
const retryable = safeError.retryable && Number(current.attempts ?? 0) < maxAttempts;
|
||||
return {
|
||||
...current,
|
||||
status: retryable ? 'retryable' : 'failed',
|
||||
reason: safeError.code,
|
||||
error: safeError,
|
||||
lease: null,
|
||||
nextAttemptAt: retryable
|
||||
? now + clampInteger(retryDelayMs, 1_000, 0, 60 * 60_000)
|
||||
: null,
|
||||
updatedAt: now,
|
||||
completedAt: retryable ? null : now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: updated.status === 'retryable'
|
||||
? 'executor_job_retry_scheduled'
|
||||
: 'executor_job_failed',
|
||||
timestamp: now,
|
||||
data: {
|
||||
status: updated.status,
|
||||
error: safeError,
|
||||
nextAttemptAt: updated.nextAttemptAt,
|
||||
},
|
||||
}),
|
||||
});
|
||||
return requireApplied(transition, 'fail');
|
||||
}
|
||||
|
||||
async function recoverExpired({ limit = 100 } = {}) {
|
||||
const now = nowMs();
|
||||
const expired = await store.listExpiredLeases({
|
||||
now,
|
||||
limit: clampInteger(limit, 100, 1, 500),
|
||||
});
|
||||
const recovered = [];
|
||||
for (const candidate of expired) {
|
||||
const leaseToken = candidate.lease?.token;
|
||||
if (!leaseToken) continue;
|
||||
const transition = await store.transition(candidate.jobId, {
|
||||
expectedStatuses: [...ACTIVE_STATUSES],
|
||||
leaseToken,
|
||||
updater(current) {
|
||||
const maxAttempts = clampInteger(
|
||||
current.maxAttempts,
|
||||
defaultMaxAttempts,
|
||||
1,
|
||||
20,
|
||||
);
|
||||
const retryable = Number(current.attempts ?? 0) < maxAttempts;
|
||||
return {
|
||||
...current,
|
||||
status: retryable ? 'retryable' : 'timed_out',
|
||||
reason: 'executor_lease_expired',
|
||||
error: {
|
||||
code: 'EXECUTOR_LEASE_EXPIRED',
|
||||
message: 'Worker lease expired before completion',
|
||||
retryable,
|
||||
},
|
||||
lease: null,
|
||||
nextAttemptAt: retryable ? now : null,
|
||||
updatedAt: now,
|
||||
completedAt: retryable ? null : now,
|
||||
};
|
||||
},
|
||||
eventFactory: (updated) => buildExecutorJobEvent({
|
||||
jobId: updated.jobId,
|
||||
type: updated.status === 'retryable'
|
||||
? 'executor_job_lease_recovered'
|
||||
: 'executor_job_timed_out',
|
||||
timestamp: now,
|
||||
data: {
|
||||
previousWorkerId: candidate.lease?.workerId ?? null,
|
||||
status: updated.status,
|
||||
attempt: updated.attempts,
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (transition.applied) recovered.push(transition.record);
|
||||
}
|
||||
return {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
inspected: expired.length,
|
||||
recovered,
|
||||
};
|
||||
}
|
||||
|
||||
async function stats() {
|
||||
const now = nowMs();
|
||||
const workers = typeof store.listWorkers === 'function'
|
||||
? await store.listWorkers({ now, staleAfterMs: workerStaleAfterMs })
|
||||
: [];
|
||||
return {
|
||||
version: WORKER_PROTOCOL_VERSION,
|
||||
...(await store.getQueueStats({ now })),
|
||||
workers,
|
||||
healthyWorkers: workers.filter((worker) => !worker.stale && !worker.draining).length,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
claim,
|
||||
start,
|
||||
heartbeat,
|
||||
progress,
|
||||
complete,
|
||||
fail,
|
||||
recoverExpired,
|
||||
stats,
|
||||
};
|
||||
}
|
||||
|
||||
export const executorWorkerProtocolInternals = {
|
||||
ACTIVE_STATUSES,
|
||||
DEFAULT_LEASE_DURATION_MS,
|
||||
DEFAULT_MAX_ATTEMPTS,
|
||||
MAX_PROGRESS_DATA_BYTES,
|
||||
WORKER_PROTOCOL_VERSION,
|
||||
clampInteger,
|
||||
protocolError,
|
||||
requiredIdentifier,
|
||||
sanitizeData,
|
||||
sanitizeError,
|
||||
sanitizeResult,
|
||||
terminalStatuses: executorGatewayInternals.TERMINAL_JOB_STATUSES,
|
||||
};
|
||||
Reference in New Issue
Block a user