feat: harden orchestrator execution runtime

agent-run-gateway.test.mjs

@@ -7,6 +7,7 @@ import test from 'node:test';
 import {
   assertRequiredImageGenerationCompleted,
   createAgentRunGateway,
+  normalizeAgentRunWorkerIdentity,
 } from './agent-run-gateway.mjs';
 
 test('required image generation cannot succeed without a verified raster image_make result', () => {
@@ -32,6 +33,21 @@ test('required image generation cannot succeed without a verified raster image_m
   }));
 });
 
+test('normalizeAgentRunWorkerIdentity creates a stable normalized runtime boundary', () => {
+  assert.deepEqual(
+    normalizeAgentRunWorkerIdentity({
+      workerId: 'worker-a',
+      runtimeRoot: '/tmp/runtime/../runtime/current',
+      buildId: 'build-123',
+    }),
+    {
+      workerId: 'worker-a',
+      runtimeRoot: '/tmp/runtime/current',
+      buildId: 'build-123',
+    },
+  );
+});
+
 function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} } = {}) {
   const runs = new Map();
   const events = [];
@@ -158,9 +174,14 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
           }))];
       }
       if (sql.includes('SELECT id') && sql.includes("status IN ('queued', 'retryable')")) {
-        const limit = Number(params[0] ?? 1);
+        const [runtimeRoot, buildId, rawLimit = 1] = params;
+        const limit = Number(rawLimit);
         return [[...runs.values()]
-          .filter((row) => ['queued', 'retryable'].includes(row.status))
+          .filter((row) => (
+            ['queued', 'retryable'].includes(row.status)
+            && (row.required_runtime_root == null || row.required_runtime_root === runtimeRoot)
+            && (row.required_build_id == null || row.required_build_id === buildId)
+          ))
           .sort((a, b) => Number(a.updated_at) - Number(b.updated_at))
           .slice(0, limit)
           .map((row) => ({ id: row.id }))];
@@ -174,6 +195,8 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
           userMessageJson,
           createdAt,
           updatedAt,
+          requiredRuntimeRoot,
+          requiredBuildId,
         ] = params;
         runs.set(id, {
           id,
@@ -188,6 +211,11 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
           updated_at: updatedAt,
           started_at: null,
           completed_at: null,
+          required_runtime_root: requiredRuntimeRoot,
+          required_build_id: requiredBuildId,
+          claimed_worker_id: null,
+          claimed_runtime_root: null,
+          claimed_build_id: null,
         });
         return [{ affectedRows: 1 }];
       }
@@ -226,9 +254,24 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
         return [filtered.slice(0, limit)];
       }
       if (sql.includes("SET status = 'running'")) {
-        const [attempts, startedAt, updatedAt, id] = params;
+        const [
+          attempts,
+          startedAt,
+          updatedAt,
+          workerId,
+          runtimeRoot,
+          buildId,
+          id,
+          requiredRuntimeRoot,
+          requiredBuildId,
+        ] = params;
         const row = runs.get(id);
-        if (!row || !['queued', 'retryable'].includes(row.status)) {
+        if (
+          !row
+          || !['queued', 'retryable'].includes(row.status)
+          || (row.required_runtime_root != null && row.required_runtime_root !== requiredRuntimeRoot)
+          || (row.required_build_id != null && row.required_build_id !== requiredBuildId)
+        ) {
           return [{ affectedRows: 0 }];
         }
         Object.assign(row, {
@@ -237,6 +280,9 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
           started_at: row.started_at ?? startedAt,
           updated_at: updatedAt,
           error_message: null,
+          claimed_worker_id: workerId,
+          claimed_runtime_root: runtimeRoot,
+          claimed_build_id: buildId,
         });
         return [{ affectedRows: 1 }];
       }
@@ -724,6 +770,48 @@ test('agent run awaits session Finish before succeeding when proxy supports it',
   assert.equal(finishEvents.length, 1);
 });
 
+test('terminal agent run quiesces session extensions after preserving Finish', async () => {
+  const pool = createFakePool();
+  const quiesced = [];
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: {
+      async startSessionForUser() {
+        return { id: 'session-quiesce-1' };
+      },
+      async submitSessionReplyAndAwaitFinishForUser() {
+        return { ok: true, finishEvent: { type: 'Finish' } };
+      },
+    },
+    quiesceSessionOnTerminal: async (input) => {
+      quiesced.push(input);
+      return { removed: ['sandbox-fs'], skipped: false };
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-quiesce-1',
+    userMessage: { role: 'user', content: [{ type: 'text', text: 'hello' }] },
+  });
+
+  await waitFor(() =>
+    pool.events.some(
+      (event) =>
+        event.runId === run.id
+        && event.eventType === 'session_extensions_quiesced',
+    ));
+  assert.equal(pool.runs.get(run.id).status, 'succeeded');
+  assert.deepEqual(quiesced, [{
+    runId: run.id,
+    userId: 'user-1',
+    sessionId: 'session-quiesce-1',
+    requestId: 'req-quiesce-1',
+    status: 'succeeded',
+  }]);
+});
+
 test('Page Data run fails closed when Finish arrives without a generated page', async () => {
   const pool = createFakePool();
   const gateway = createAgentRunGateway({
@@ -2008,6 +2096,56 @@ test('external worker dispatches queued runs through the same queue controls', a
   assert.deepEqual(submitted, ['req-worker']);
 });
 
+test('external worker cannot dispatch a run targeted at another runtime identity', async () => {
+  const pool = createFakePool();
+  const proxy = {
+    async startSessionForUser() {
+      return { id: 'session-runtime-boundary' };
+    },
+    async submitSessionReplyForUser() {},
+  };
+  const owner = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: proxy,
+    autoDispatch: false,
+    retryDelaysMs: [],
+    workerIdentity: {
+      workerId: 'worker-current',
+      runtimeRoot: '/srv/memind/current',
+      buildId: 'build-current',
+    },
+  });
+  const stale = createAgentRunGateway({
+    pool,
+    userAuth: {},
+    tkmindProxy: proxy,
+    autoDispatch: false,
+    retryDelaysMs: [],
+    workerIdentity: {
+      workerId: 'worker-stale',
+      runtimeRoot: '/private/tmp/memind-stale',
+      buildId: 'build-stale',
+    },
+  });
+
+  const run = await owner.createRun('user-1', {
+    requestId: 'req-runtime-boundary',
+    userMessage: { role: 'user', content: [] },
+  });
+
+  const staleResult = await stale.dispatchQueuedRuns({ limit: 10 });
+  assert.equal(staleResult.dispatched, 0);
+  assert.equal(pool.runs.get(run.id).status, 'queued');
+
+  const ownerResult = await owner.dispatchQueuedRuns({ limit: 10 });
+  assert.equal(ownerResult.dispatched, 1);
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  assert.equal(pool.runs.get(run.id).claimed_worker_id, 'worker-current');
+  assert.equal(pool.runs.get(run.id).claimed_runtime_root, '/srv/memind/current');
+  assert.equal(pool.runs.get(run.id).claimed_build_id, 'build-current');
+});
+
 test('external worker does not dispatch more runs when local queue is full', async () => {
   const pool = createFakePool();
   const release = [];