feat: persist blocked executor jobs

services/orchestrator/server.mjs

@@ -1,6 +1,7 @@
 import { pathToFileURL } from 'node:url';
 import { createOrchestratorApp } from './app.mjs';
 import { createOrchestratorCheckpoint } from './checkpoint.mjs';
+import { createExecutorJobPersistence } from './executor-job-store.mjs';
 import { createLangGraphOrchestratorRuntime } from './runtime.mjs';
 
 function positivePort(value, fallback = 8093) {
@@ -18,11 +19,24 @@ export async function startOrchestratorServer({
     connectionString: env.MEMIND_ORCHESTRATOR_DATABASE_URL,
     schema: env.MEMIND_ORCHESTRATOR_DATABASE_SCHEMA,
   });
+  let executorJobs;
+  try {
+    executorJobs = await createExecutorJobPersistence({
+      mode: env.MEMIND_ORCHESTRATOR_CHECKPOINT_MODE,
+      connectionString: env.MEMIND_ORCHESTRATOR_DATABASE_URL,
+      schema: env.MEMIND_ORCHESTRATOR_DATABASE_SCHEMA,
+    });
+  } catch (error) {
+    await Promise.allSettled([checkpoint.close()]);
+    throw error;
+  }
   const runtime = createLangGraphOrchestratorRuntime({
     checkpointer: checkpoint.checkpointer,
     checkpointKind: checkpoint.kind,
     durable: checkpoint.durable,
     checkpointProbe: checkpoint.probe,
+    executorJobStore: executorJobs.store,
+    executorJobProbe: executorJobs.probe,
   });
   const app = createOrchestratorApp({
     runtime,
@@ -30,20 +44,44 @@ export async function startOrchestratorServer({
   });
   const host = String(env.MEMIND_ORCHESTRATOR_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
   const port = positivePort(env.MEMIND_ORCHESTRATOR_PORT, 8093);
-  const server = await new Promise((resolve, reject) => {
-    const listening = app.listen(port, host, () => resolve(listening));
-    listening.once('error', reject);
-  });
+  let server;
+  try {
+    server = await new Promise((resolve, reject) => {
+      const listening = app.listen(port, host, () => resolve(listening));
+      listening.once('error', reject);
+    });
+  } catch (error) {
+    await Promise.allSettled([executorJobs.close(), checkpoint.close()]);
+    throw error;
+  }
   logger.log(`[orchestrator] listening on http://${host}:${port} (${checkpoint.kind})`);
 
   async function close() {
-    await new Promise((resolve, reject) => {
-      server.close((error) => (error ? reject(error) : resolve()));
-    });
-    await checkpoint.close();
+    let serverCloseError = null;
+    try {
+      await new Promise((resolve, reject) => {
+        server.close((error) => (error ? reject(error) : resolve()));
+      });
+    } catch (error) {
+      serverCloseError = error;
+    }
+    const results = await Promise.allSettled([
+      executorJobs.close(),
+      checkpoint.close(),
+    ]);
+    if (serverCloseError) throw serverCloseError;
+    const failed = results.find((result) => result.status === 'rejected');
+    if (failed) throw failed.reason;
   }
 
-  return { app, server, runtime, checkpoint, close };
+  return {
+    app,
+    server,
+    runtime,
+    checkpoint,
+    executorJobs,
+    close,
+  };
 }
 
 const isEntrypoint = process.argv[1]