feat: harden orchestrator execution runtime
This commit is contained in:
john
@@ -0,0 +1,274 @@
|
||||
function workerError(code, message, options = {}) {
|
||||
const error = new Error(message);
|
||||
error.code = code;
|
||||
error.retryable = options.retryable === true;
|
||||
return error;
|
||||
}
|
||||
|
||||
function safeJson(response) {
|
||||
return response.json().catch(() => null);
|
||||
}
|
||||
|
||||
export function createExecutorWorkerClient({
|
||||
baseUrl,
|
||||
serviceToken = '',
|
||||
workerToken = '',
|
||||
fetchImpl = globalThis.fetch,
|
||||
} = {}) {
|
||||
const target = String(baseUrl ?? '').trim().replace(/\/$/, '');
|
||||
if (!target) throw new Error('Executor worker client requires baseUrl');
|
||||
if (typeof fetchImpl !== 'function') throw new Error('Executor worker client requires fetch');
|
||||
|
||||
async function request(path, { method = 'GET', body = null } = {}) {
|
||||
const response = await fetchImpl(`${target}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
...(body == null ? {} : { 'content-type': 'application/json' }),
|
||||
...(serviceToken ? { authorization: `Bearer ${serviceToken}` } : {}),
|
||||
...(workerToken ? { 'x-orchestrator-worker-token': workerToken } : {}),
|
||||
},
|
||||
...(body == null ? {} : { body: JSON.stringify(body) }),
|
||||
});
|
||||
if (response.status === 204) return null;
|
||||
const payload = await safeJson(response);
|
||||
if (!response.ok) {
|
||||
const error = workerError(
|
||||
payload?.error?.code ?? 'EXECUTOR_WORKER_HTTP_ERROR',
|
||||
payload?.error?.message ?? `Executor worker request failed (${response.status})`,
|
||||
{ retryable: response.status >= 500 || response.status === 429 },
|
||||
);
|
||||
error.status = response.status;
|
||||
throw error;
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
|
||||
return {
|
||||
claim(input) {
|
||||
return request('/v1/workers/claim', { method: 'POST', body: input });
|
||||
},
|
||||
start(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/start`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
heartbeat(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/heartbeat`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
progress(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/progress`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
complete(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/complete`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
fail(jobId, input) {
|
||||
return request(`/v1/workers/jobs/${encodeURIComponent(jobId)}/fail`, {
|
||||
method: 'POST',
|
||||
body: input,
|
||||
});
|
||||
},
|
||||
recoverExpired(input = {}) {
|
||||
return request('/v1/workers/recover-expired', { method: 'POST', body: input });
|
||||
},
|
||||
stats() {
|
||||
return request('/v1/workers/stats');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerAdapterRegistry(initialAdapters = []) {
|
||||
const adapters = new Map();
|
||||
return {
|
||||
register(adapter) {
|
||||
const id = String(adapter?.id ?? '').trim().toLowerCase();
|
||||
if (!id || typeof adapter?.submit !== 'function') {
|
||||
throw new Error('Worker adapter requires id and submit');
|
||||
}
|
||||
if (adapters.has(id)) throw new Error(`Worker adapter already registered: ${id}`);
|
||||
adapters.set(id, adapter);
|
||||
return adapter;
|
||||
},
|
||||
get(id) {
|
||||
return adapters.get(String(id ?? '').trim().toLowerCase()) ?? null;
|
||||
},
|
||||
list() {
|
||||
return [...adapters.values()].map((adapter) => ({
|
||||
id: adapter.id,
|
||||
kind: adapter.kind ?? 'worker',
|
||||
capabilities: [...(adapter.capabilities ?? [])],
|
||||
}));
|
||||
},
|
||||
ids() {
|
||||
return [...adapters.keys()];
|
||||
},
|
||||
_initialize() {
|
||||
for (const adapter of initialAdapters) this.register(adapter);
|
||||
return this;
|
||||
},
|
||||
}._initialize();
|
||||
}
|
||||
|
||||
export function createExecutorWorkerRuntime({
|
||||
client,
|
||||
registry = createWorkerAdapterRegistry(),
|
||||
workerId,
|
||||
heartbeatMs = 10_000,
|
||||
leaseDurationMs = 30_000,
|
||||
logger = console,
|
||||
} = {}) {
|
||||
const normalizedWorkerId = String(workerId ?? '').trim();
|
||||
if (!normalizedWorkerId) throw new Error('Executor worker runtime requires workerId');
|
||||
if (!client?.claim) throw new Error('Executor worker runtime requires client');
|
||||
let stopping = false;
|
||||
const active = new Map();
|
||||
|
||||
async function executeAdapter(adapter, job, leaseToken, controller) {
|
||||
return adapter.submit(job.request, {
|
||||
jobId: job.jobId,
|
||||
signal: controller.signal,
|
||||
emit(type, data) {
|
||||
return client.progress(job.jobId, { leaseToken, type, data });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function processClaim(claim) {
|
||||
const job = claim.job;
|
||||
const leaseToken = claim.leaseToken;
|
||||
const controller = new AbortController();
|
||||
active.set(job.jobId, { controller, leaseToken });
|
||||
const timeoutMs = Math.max(500, Number(job.timeoutMs ?? job.request?.controls?.timeoutMs) || 900_000);
|
||||
const timeout = setTimeout(() => controller.abort('executor_timeout'), timeoutMs);
|
||||
let heartbeatFailure = null;
|
||||
const heartbeat = setInterval(() => {
|
||||
void client.heartbeat(job.jobId, { leaseToken, leaseDurationMs }).catch((error) => {
|
||||
heartbeatFailure = error;
|
||||
controller.abort('heartbeat_failed');
|
||||
});
|
||||
}, Math.max(1_000, Math.min(heartbeatMs, Math.floor(leaseDurationMs / 2))));
|
||||
heartbeat.unref?.();
|
||||
try {
|
||||
await client.start(job.jobId, { leaseToken });
|
||||
if (!job.request) {
|
||||
throw workerError(
|
||||
'EXECUTOR_JOB_REQUEST_MISSING',
|
||||
'Claimed executor job does not contain an execution request',
|
||||
);
|
||||
}
|
||||
const primary = registry.get(job.executor);
|
||||
if (!primary) {
|
||||
throw workerError(
|
||||
'EXECUTOR_ADAPTER_NOT_INSTALLED',
|
||||
`Worker does not have adapter ${job.executor}`,
|
||||
);
|
||||
}
|
||||
let result;
|
||||
try {
|
||||
result = await executeAdapter(primary, job, leaseToken, controller);
|
||||
} catch (primaryError) {
|
||||
const fallbackId = String(job.fallbackExecutor ?? '').trim();
|
||||
const fallback = job.fallbackAvailable ? registry.get(fallbackId) : null;
|
||||
if (!fallback || controller.signal.aborted) throw primaryError;
|
||||
await client.progress(job.jobId, {
|
||||
leaseToken,
|
||||
type: 'executor_job_fallback_started',
|
||||
data: {
|
||||
from: job.executor,
|
||||
to: fallbackId,
|
||||
reason: String(primaryError?.code ?? 'EXECUTOR_PRIMARY_FAILED').slice(0, 128),
|
||||
},
|
||||
});
|
||||
result = await executeAdapter(fallback, {
|
||||
...job,
|
||||
executor: fallbackId,
|
||||
request: { ...job.request, executor: fallbackId },
|
||||
}, leaseToken, controller);
|
||||
}
|
||||
if (heartbeatFailure) throw heartbeatFailure;
|
||||
await client.complete(job.jobId, { leaseToken, result });
|
||||
return { processed: true, jobId: job.jobId, status: 'succeeded' };
|
||||
} catch (error) {
|
||||
const failure = controller.signal.aborted && !heartbeatFailure
|
||||
? {
|
||||
code: 'EXECUTOR_TIMEOUT_OR_CANCELLED',
|
||||
message: 'Executor was cancelled or exceeded its timeout',
|
||||
retryable: false,
|
||||
}
|
||||
: {
|
||||
code: String(error?.code ?? 'EXECUTOR_WORKER_FAILED').slice(0, 128),
|
||||
message: String(error?.message ?? error ?? 'Executor worker failed').slice(0, 2048),
|
||||
retryable: error?.retryable === true,
|
||||
};
|
||||
try {
|
||||
await client.fail(job.jobId, {
|
||||
leaseToken,
|
||||
error: failure,
|
||||
retryDelayMs: failure.retryable ? 2_000 : 0,
|
||||
});
|
||||
} catch (reportError) {
|
||||
logger.error?.(
|
||||
`[executor-worker] failed to report ${job.jobId}: ${reportError.message}`,
|
||||
);
|
||||
throw reportError;
|
||||
}
|
||||
return { processed: true, jobId: job.jobId, status: 'failed', error: failure };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
clearInterval(heartbeat);
|
||||
active.delete(job.jobId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async runOnce() {
|
||||
if (stopping) return { processed: false, reason: 'stopping' };
|
||||
const claim = await client.claim({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: registry.ids(),
|
||||
leaseDurationMs,
|
||||
});
|
||||
if (!claim) return { processed: false, reason: 'queue_empty' };
|
||||
return processClaim(claim);
|
||||
},
|
||||
async recoverExpired(options = {}) {
|
||||
return client.recoverExpired(options);
|
||||
},
|
||||
status() {
|
||||
return {
|
||||
version: 'executor-worker-runtime-v1',
|
||||
workerId: normalizedWorkerId,
|
||||
stopping,
|
||||
activeJobs: [...active.keys()],
|
||||
adapters: registry.list(),
|
||||
heartbeatMs,
|
||||
leaseDurationMs,
|
||||
};
|
||||
},
|
||||
async stop() {
|
||||
stopping = true;
|
||||
await client.claim({
|
||||
workerId: normalizedWorkerId,
|
||||
executors: registry.ids(),
|
||||
leaseDurationMs,
|
||||
draining: true,
|
||||
}).catch(() => null);
|
||||
for (const { controller } of active.values()) controller.abort('worker_stopping');
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const executorWorkerRuntimeInternals = {
|
||||
safeJson,
|
||||
workerError,
|
||||
};
|
||||
Reference in New Issue
Block a user