feat: harden orchestrator execution runtime

services/orchestrator/runtime.mjs

@@ -1,9 +1,15 @@
-import { normalizeRunSpec } from './contracts.mjs';
+import { buildRunEvent, normalizeRunSpec } from './contracts.mjs';
 import {
   CODE_RUN_SHADOW_WORKFLOW,
   createCodeRunShadowGraph,
 } from './code-run-shadow-graph.mjs';
-import { createExecutorGateway } from './executor-gateway.mjs';
+import {
+  createExecutorGateway,
+  createInMemoryExecutorJobStore,
+  createPhase5ExecutorAdapterRegistry,
+} from './executor-gateway.mjs';
+import { createExecutorWorkerCoordinator } from './executor-worker-protocol.mjs';
+import { createExecutorAdmissionPolicy } from './executor-admission-policy.mjs';
 
 function graphConfig(runId) {
   return {
@@ -38,11 +44,32 @@ export function createLangGraphOrchestratorRuntime({
   checkpointProbe = null,
   executorJobStore = null,
   executorJobProbe = null,
+  executionEnabled = false,
+  enabledExecutors = [],
+  workerOptions = {},
+  admissionConfig = {},
 } = {}) {
-  const executorGateway = createExecutorGateway({
-    ...(executorJobStore ? { store: executorJobStore } : {}),
+  const jobStore = executorJobStore ?? createInMemoryExecutorJobStore();
+  const admissionPolicy = createExecutorAdmissionPolicy({
+    store: jobStore,
+    config: admissionConfig,
+    nowMs: workerOptions.nowMs,
+  });
+  const executorGateway = createExecutorGateway({
+    store: jobStore,
+    registry: createPhase5ExecutorAdapterRegistry({ enabledExecutors }),
+    executionEnabled,
+    admissionPolicy,
+  });
+  const workerCoordinator = createExecutorWorkerCoordinator({
+    store: jobStore,
+    ...workerOptions,
+  });
+  const graph = createCodeRunShadowGraph({
+    checkpointer,
+    executorGateway,
+    executionEnabled,
   });
-  const graph = createCodeRunShadowGraph({ checkpointer, executorGateway });
   const probe = typeof checkpointProbe === 'function'
     ? checkpointProbe
     : async () => {
@@ -57,10 +84,54 @@ export function createLangGraphOrchestratorRuntime({
     return graph.getState(graphConfig(runId));
   }
 
+  async function refreshExecutorTerminal(runId, snapshot) {
+    const values = snapshot?.values ?? {};
+    if (values.status !== 'waiting') return snapshot;
+    const jobId = values.plan?.executorJob?.id;
+    if (!jobId) return snapshot;
+    const job = await executorGateway.getJob(jobId);
+    const terminal = {
+      succeeded: { status: 'succeeded', phase: 'completed' },
+      failed: { status: 'failed', phase: 'executor_failed' },
+      cancelled: { status: 'cancelled', phase: 'cancelled' },
+      timed_out: { status: 'failed', phase: 'executor_timed_out' },
+    }[job?.status];
+    if (!terminal) return snapshot;
+    const result = {
+      observed: false,
+      executed: job.status === 'succeeded',
+      executorAdapter: values.plan.executorAdapter,
+      taskType: values.plan.taskType,
+      executorJob: {
+        id: job.jobId,
+        status: job.status,
+        reason: job.reason,
+        result: job.result ?? null,
+      },
+    };
+    await graph.updateState(graphConfig(runId), {
+      ...terminal,
+      result,
+      events: [buildRunEvent({
+        runId,
+        sequence: (values.events?.length ?? 0) + 1,
+        type: job.status === 'succeeded'
+          ? 'workflow_completed'
+          : `workflow_${job.status}`,
+        data: result,
+      })],
+    });
+    return getSnapshot(runId);
+  }
+
   async function getState(runId) {
     const normalizedRunId = String(runId ?? '').trim();
     if (!normalizedRunId) return null;
-    return projectSnapshot(normalizedRunId, await getSnapshot(normalizedRunId));
+    const snapshot = await getSnapshot(normalizedRunId);
+    return projectSnapshot(
+      normalizedRunId,
+      await refreshExecutorTerminal(normalizedRunId, snapshot),
+    );
   }
 
   function health() {
@@ -71,8 +142,13 @@ export function createLangGraphOrchestratorRuntime({
         kind: checkpointKind,
         durable: Boolean(durable),
       },
-      execution: 'observe-only',
+      execution: executionEnabled ? 'worker-queue-enabled' : 'observe-only',
       executorGateway: executorGateway.status(),
+      workerProtocol: {
+        version: 'executor-worker-v1',
+        enabled: executionEnabled === true,
+      },
+      admissionPolicy: admissionPolicy.status(),
     };
   }
 
@@ -100,7 +176,10 @@ export function createLangGraphOrchestratorRuntime({
 
     async listEvents(runId, { after = 0 } = {}) {
       const normalizedRunId = String(runId ?? '').trim();
-      const snapshot = normalizedRunId ? await getSnapshot(normalizedRunId) : null;
+      const initial = normalizedRunId ? await getSnapshot(normalizedRunId) : null;
+      const snapshot = normalizedRunId
+        ? await refreshExecutorTerminal(normalizedRunId, initial)
+        : null;
       if (!snapshot?.values?.spec) return null;
       const cursor = Math.max(0, Number(after) || 0);
       const events = (snapshot.values.events ?? []).filter((event) => event.sequence > cursor);
@@ -114,6 +193,7 @@ export function createLangGraphOrchestratorRuntime({
     async resume(runId) {
       const state = await getState(runId);
       if (!state) return null;
+      if (state.status === 'waiting') return state;
       const error = new Error('This workflow has no interrupt point to resume');
       error.code = 'WORKFLOW_NOT_INTERRUPTED';
       error.status = 409;
@@ -123,6 +203,12 @@ export function createLangGraphOrchestratorRuntime({
     async cancel(runId) {
       const state = await getState(runId);
       if (!state) return null;
+      if (state.status === 'waiting' && state.plan?.executorJob?.id) {
+        await executorGateway.cancel(state.plan.executorJob.id, {
+          reason: 'workflow_cancelled',
+        });
+        return getState(runId);
+      }
       const error = new Error('Completed shadow observations cannot be cancelled');
       error.code = 'WORKFLOW_ALREADY_TERMINAL';
       error.status = 409;
@@ -137,13 +223,69 @@ export function createLangGraphOrchestratorRuntime({
       return executorGateway.listEvents(jobId, options);
     },
 
+    submitExecutorJob(input) {
+      return executorGateway.createJob(input);
+    },
+
+    cancelExecutorJob(jobId, options = {}) {
+      return executorGateway.cancel(jobId, options);
+    },
+
+    claimExecutorJob(input) {
+      return workerCoordinator.claim(input);
+    },
+
+    startExecutorJob(jobId, input) {
+      return workerCoordinator.start(jobId, input);
+    },
+
+    heartbeatExecutorJob(jobId, input) {
+      return workerCoordinator.heartbeat(jobId, input);
+    },
+
+    progressExecutorJob(jobId, input) {
+      return workerCoordinator.progress(jobId, input);
+    },
+
+    completeExecutorJob(jobId, input) {
+      return workerCoordinator.complete(jobId, input);
+    },
+
+    failExecutorJob(jobId, input) {
+      return workerCoordinator.fail(jobId, input);
+    },
+
+    recoverExpiredExecutorJobs(input = {}) {
+      return workerCoordinator.recoverExpired(input);
+    },
+
+    getExecutorQueueStats() {
+      return workerCoordinator.stats();
+    },
+
+    async metrics() {
+      return {
+        version: 'orchestrator-metrics-v1',
+        timestamp: Date.now(),
+        queue: await workerCoordinator.stats(),
+        gateway: executorGateway.status(),
+      };
+    },
+
     health,
 
     async ready() {
       await Promise.all([probe(), probeExecutorJobs()]);
+      const executorQueue = await workerCoordinator.stats();
+      if (executionEnabled && executorQueue.healthyWorkers < 1) {
+        const error = new Error('Execution is enabled but no healthy executor worker is registered');
+        error.code = 'EXECUTOR_WORKER_UNAVAILABLE';
+        throw error;
+      }
       return {
         ...health(),
         ready: true,
+        executorQueue,
       };
     },
   };