feat: expose executor job observability

services/orchestrator/executor-gateway.mjs

@@ -3,8 +3,10 @@ import { normalizeWorkflowEngineId } from './contracts.mjs';
 
 const EXECUTOR_JOB_REQUEST_VERSION = 'executor-job-request-v1';
 const EXECUTOR_JOB_STATE_VERSION = 'executor-job-state-v1';
+const EXECUTOR_JOB_EVENT_VERSION = 'executor-job-event-v1';
 const EXECUTOR_DISPATCH_IMPLEMENTED = false;
 const MAX_INSTRUCTION_CHARACTERS = 32_000;
+const MAX_EVENT_PAGE_SIZE = 500;
 const TERMINAL_JOB_STATUSES = new Set([
   'succeeded',
   'failed',
@@ -77,6 +79,33 @@ function gatewayError(code, message, status = 409) {
   return error;
 }
 
+export function buildExecutorJobEvent({
+  eventId = crypto.randomUUID(),
+  jobId,
+  sequence = 0,
+  type,
+  timestamp = Date.now(),
+  data = null,
+} = {}) {
+  const normalizedJobId = String(jobId ?? '').trim();
+  const normalizedType = String(type ?? '').trim();
+  if (!normalizedJobId) {
+    throw gatewayError('EXECUTOR_EVENT_JOB_ID_REQUIRED', 'Executor event requires jobId', 422);
+  }
+  if (!normalizedType) {
+    throw gatewayError('EXECUTOR_EVENT_TYPE_REQUIRED', 'Executor event requires type', 422);
+  }
+  return {
+    version: EXECUTOR_JOB_EVENT_VERSION,
+    eventId: String(eventId),
+    jobId: normalizedJobId,
+    sequence: Math.max(0, Number(sequence) || 0),
+    type: normalizedType,
+    timestamp: Number(timestamp) || Date.now(),
+    data: data == null ? null : clone(data),
+  };
+}
+
 export function normalizeExecutorJobRequest(input = {}) {
   const source = input && typeof input === 'object' && !Array.isArray(input) ? input : {};
   const jobId = String(source.jobId ?? crypto.randomUUID()).trim();
@@ -86,6 +115,13 @@ export function normalizeExecutorJobRequest(input = {}) {
     .trim()
     .slice(0, MAX_INSTRUCTION_CHARACTERS);
   if (!jobId) throw gatewayError('EXECUTOR_JOB_ID_REQUIRED', 'Executor job requires jobId', 422);
+  if (!/^[a-zA-Z0-9:_-]{1,128}$/.test(jobId)) {
+    throw gatewayError(
+      'EXECUTOR_JOB_ID_INVALID',
+      'Executor jobId must be a safe identifier no longer than 128 characters',
+      422,
+    );
+  }
   if (!idempotencyKey) {
     throw gatewayError(
       'EXECUTOR_IDEMPOTENCY_KEY_REQUIRED',
@@ -229,6 +265,19 @@ export function listPhase3ExecutorAdapters() {
 export function createInMemoryExecutorJobStore() {
   const byId = new Map();
   const byIdempotencyKey = new Map();
+  const eventsByJobId = new Map();
+
+  function appendEvent(jobId, event) {
+    if (!event) return;
+    const events = eventsByJobId.get(jobId) ?? [];
+    events.push({
+      ...clone(event),
+      jobId,
+      sequence: events.length + 1,
+    });
+    eventsByJobId.set(jobId, events);
+  }
+
   return {
     kind: 'memory',
     durable: false,
@@ -241,7 +290,7 @@ export function createInMemoryExecutorJobStore() {
       const value = jobId ? byId.get(jobId) : null;
       return value ? clone(value) : null;
     },
-    async createIfAbsent(record) {
+    async createIfAbsent(record, { initialEvent = null } = {}) {
       const existingId = byIdempotencyKey.get(record.idempotencyKey);
       if (existingId) return { created: false, record: clone(byId.get(existingId)) };
       if (byId.has(record.jobId)) {
@@ -249,18 +298,34 @@ export function createInMemoryExecutorJobStore() {
       }
       byId.set(record.jobId, clone(record));
       byIdempotencyKey.set(record.idempotencyKey, record.jobId);
+      appendEvent(record.jobId, initialEvent);
       return { created: true, record: clone(record) };
     },
-    async update(record) {
+    async update(record, { event = null } = {}) {
       if (!byId.has(record.jobId)) return null;
       byId.set(record.jobId, clone(record));
+      appendEvent(record.jobId, event);
       return clone(record);
     },
+    async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
+      const cursor = Math.max(0, Number(after) || 0);
+      const pageSize = clampInteger(limit, MAX_EVENT_PAGE_SIZE, 1, MAX_EVENT_PAGE_SIZE);
+      return (eventsByJobId.get(String(jobId ?? '').trim()) ?? [])
+        .filter((event) => event.sequence > cursor)
+        .slice(0, pageSize)
+        .map(clone);
+    },
   };
 }
 
 function validateJobStore(store) {
-  for (const method of ['getById', 'getByIdempotencyKey', 'createIfAbsent', 'update']) {
+  for (const method of [
+    'getById',
+    'getByIdempotencyKey',
+    'createIfAbsent',
+    'update',
+    'listEvents',
+  ]) {
     if (typeof store?.[method] !== 'function') {
       throw new Error(`Executor job store missing method: ${method}`);
     }
@@ -354,7 +419,20 @@ export function createExecutorGateway({
       startedAt: null,
       completedAt: now,
     };
-    const result = await store.createIfAbsent(record);
+    const result = await store.createIfAbsent(record, {
+      initialEvent: buildExecutorJobEvent({
+        jobId: request.jobId,
+        sequence: 1,
+        type: 'executor_job_blocked',
+        timestamp: now,
+        data: {
+          executor: request.executor,
+          status: record.status,
+          reason: record.reason,
+          dispatchAllowed: false,
+        },
+      }),
+    });
     if (
       !result.created
       && result.record.requestFingerprint !== requestFingerprint
@@ -373,12 +451,23 @@ export function createExecutorGateway({
     if (current.status === 'cancelled' || !current.cancellationAllowed) return current;
     if (TERMINAL_JOB_STATUSES.has(current.status) && current.status !== 'blocked') return current;
     const now = nowMs();
-    return store.update({
+    const updated = {
       ...current,
       status: 'cancelled',
       reason: String(reason).slice(0, 256),
       updatedAt: now,
       completedAt: now,
+    };
+    return store.update(updated, {
+      event: buildExecutorJobEvent({
+        jobId: current.jobId,
+        type: 'executor_job_cancelled',
+        timestamp: now,
+        data: {
+          status: updated.status,
+          reason: updated.reason,
+        },
+      }),
     });
   }
 
@@ -389,6 +478,22 @@ export function createExecutorGateway({
     getJob(jobId) {
       return store.getById(jobId);
     },
+    async listEvents(jobId, { after = 0, limit = MAX_EVENT_PAGE_SIZE } = {}) {
+      const normalizedJobId = String(jobId ?? '').trim();
+      if (!normalizedJobId) return null;
+      const job = await store.getById(normalizedJobId);
+      if (!job) return null;
+      const cursor = Math.max(0, Number(after) || 0);
+      const events = await store.listEvents(normalizedJobId, {
+        after: cursor,
+        limit: clampInteger(limit, MAX_EVENT_PAGE_SIZE, 1, MAX_EVENT_PAGE_SIZE),
+      });
+      return {
+        jobId: normalizedJobId,
+        events,
+        nextCursor: events.at(-1)?.sequence ?? cursor,
+      };
+    },
     listAdapters() {
       return registry.list();
     },
@@ -409,8 +514,10 @@ export function createExecutorGateway({
 
 export const executorGatewayInternals = {
   EXECUTOR_DISPATCH_IMPLEMENTED,
+  EXECUTOR_JOB_EVENT_VERSION,
   EXECUTOR_JOB_REQUEST_VERSION,
   EXECUTOR_JOB_STATE_VERSION,
+  MAX_EVENT_PAGE_SIZE,
   MAX_INSTRUCTION_CHARACTERS,
   PHASE3_EXECUTOR_DESCRIPTORS,
   TERMINAL_JOB_STATUSES,