feat: harden orchestrator execution runtime

services/orchestrator/app.mjs

@@ -22,9 +22,49 @@ function authorize(serviceToken) {
   };
 }
 
+function authorizeWorker(workerToken) {
+  const expected = String(workerToken ?? '').trim();
+  return (request, response, next) => {
+    if (!expected) return next();
+    if (request.get('x-orchestrator-worker-token') === expected) return next();
+    return response.status(401).json({
+      error: {
+        code: 'WORKER_UNAUTHORIZED',
+        message: 'Missing or invalid worker token',
+      },
+    });
+  };
+}
+
+function prometheusMetrics(metrics) {
+  const lines = [
+    '# HELP memind_orchestrator_executor_jobs Executor jobs by state.',
+    '# TYPE memind_orchestrator_executor_jobs gauge',
+  ];
+  for (const [status, count] of Object.entries(metrics.queue?.counts ?? {})) {
+    lines.push(`memind_orchestrator_executor_jobs{status="${status}"} ${Number(count) || 0}`);
+  }
+  lines.push(
+    '# HELP memind_orchestrator_executor_claimable Claimable executor jobs.',
+    '# TYPE memind_orchestrator_executor_claimable gauge',
+    `memind_orchestrator_executor_claimable ${Number(metrics.queue?.claimable) || 0}`,
+    '# HELP memind_orchestrator_executor_expired_leases Expired executor leases.',
+    '# TYPE memind_orchestrator_executor_expired_leases gauge',
+    `memind_orchestrator_executor_expired_leases ${Number(metrics.queue?.expiredLeases) || 0}`,
+    '# HELP memind_orchestrator_executor_workers Healthy registered workers.',
+    '# TYPE memind_orchestrator_executor_workers gauge',
+    `memind_orchestrator_executor_workers ${Number(metrics.queue?.healthyWorkers) || 0}`,
+    '# HELP memind_orchestrator_execution_enabled Whether executor queue admission is enabled.',
+    '# TYPE memind_orchestrator_execution_enabled gauge',
+    `memind_orchestrator_execution_enabled ${metrics.gateway?.executionEnabled ? 1 : 0}`,
+  );
+  return `${lines.join('\n')}\n`;
+}
+
 export function createOrchestratorApp({
   runtime,
   serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
+  workerToken = process.env.MEMIND_ORCHESTRATOR_WORKER_TOKEN,
 } = {}) {
   if (!runtime) throw new Error('Orchestrator app requires runtime');
   const app = express();
@@ -42,6 +82,16 @@ export function createOrchestratorApp({
       });
     }
   });
+  app.get('/metrics', async (_request, response, next) => {
+    try {
+      const metrics = await runtime.metrics();
+      return response
+        .type('text/plain; version=0.0.4; charset=utf-8')
+        .send(prometheusMetrics(metrics));
+    } catch (error) {
+      return next(error);
+    }
+  });
 
   app.use('/v1', authorize(serviceToken));
 
@@ -53,6 +103,15 @@ export function createOrchestratorApp({
     }
   });
 
+  app.post('/v1/executor-jobs', async (request, response, next) => {
+    try {
+      const result = await runtime.submitExecutorJob(request.body);
+      return response.status(result.created ? 202 : 200).json(result);
+    } catch (error) {
+      return next(error);
+    }
+  });
+
   app.get('/v1/runs/:runId', async (request, response, next) => {
     try {
       const state = await runtime.getState(request.params.runId);
@@ -122,6 +181,105 @@ export function createOrchestratorApp({
     }
   });
 
+  const requireWorker = authorizeWorker(workerToken);
+
+  app.post('/v1/workers/claim', requireWorker, async (request, response, next) => {
+    try {
+      const claim = await runtime.claimExecutorJob(request.body);
+      return claim ? response.json(claim) : response.status(204).end();
+    } catch (error) {
+      return next(error);
+    }
+  });
+
+  app.post(
+    '/v1/workers/jobs/:jobId/start',
+    requireWorker,
+    async (request, response, next) => {
+      try {
+        return response.json(await runtime.startExecutorJob(request.params.jobId, request.body));
+      } catch (error) {
+        return next(error);
+      }
+    },
+  );
+
+  app.post(
+    '/v1/workers/jobs/:jobId/heartbeat',
+    requireWorker,
+    async (request, response, next) => {
+      try {
+        return response.json(await runtime.heartbeatExecutorJob(request.params.jobId, request.body));
+      } catch (error) {
+        return next(error);
+      }
+    },
+  );
+
+  app.post(
+    '/v1/workers/jobs/:jobId/progress',
+    requireWorker,
+    async (request, response, next) => {
+      try {
+        return response.json(await runtime.progressExecutorJob(request.params.jobId, request.body));
+      } catch (error) {
+        return next(error);
+      }
+    },
+  );
+
+  app.post(
+    '/v1/workers/jobs/:jobId/complete',
+    requireWorker,
+    async (request, response, next) => {
+      try {
+        return response.json(await runtime.completeExecutorJob(request.params.jobId, request.body));
+      } catch (error) {
+        return next(error);
+      }
+    },
+  );
+
+  app.post(
+    '/v1/workers/jobs/:jobId/fail',
+    requireWorker,
+    async (request, response, next) => {
+      try {
+        return response.json(await runtime.failExecutorJob(request.params.jobId, request.body));
+      } catch (error) {
+        return next(error);
+      }
+    },
+  );
+
+  app.post('/v1/workers/recover-expired', requireWorker, async (request, response, next) => {
+    try {
+      return response.json(await runtime.recoverExpiredExecutorJobs(request.body));
+    } catch (error) {
+      return next(error);
+    }
+  });
+
+  app.get('/v1/workers/stats', requireWorker, async (_request, response, next) => {
+    try {
+      return response.json(await runtime.getExecutorQueueStats());
+    } catch (error) {
+      return next(error);
+    }
+  });
+
+  app.post('/v1/executor-jobs/:jobId/cancel', async (request, response, next) => {
+    try {
+      const job = await runtime.cancelExecutorJob(request.params.jobId, request.body);
+      if (!job) {
+        return response.status(404).json({ error: { code: 'EXECUTOR_JOB_NOT_FOUND' } });
+      }
+      return response.json(job);
+    } catch (error) {
+      return next(error);
+    }
+  });
+
   app.use((error, _request, response, _next) => {
     response.status(errorStatus(error)).json({
       error: {
@@ -136,5 +294,7 @@ export function createOrchestratorApp({
 
 export const orchestratorAppInternals = {
   authorize,
+  authorizeWorker,
   errorStatus,
+  prometheusMetrics,
 };