feat: validate tool gateway artifacts

agent-run-gateway.mjs

@@ -1,4 +1,6 @@
 import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
 
 const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000];
 const TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
@@ -6,6 +8,7 @@ const CODE_TOOL_MODES = new Set(['code', 'code-task', 'code_task', 'code-tool',
 const RUN_METADATA_KEY = 'memindRun';
 const DEFAULT_MAX_CONCURRENT_RUNS = 1;
 const DEFAULT_RUN_TIMEOUT_MS = 15 * 60 * 1000;
+const TOOL_GATEWAY_SUMMARY_LIMIT = 4096;
 
 function nowMs() {
   return Date.now();
@@ -53,6 +56,103 @@ function normalizeTaskType(value) {
   return normalized || null;
 }
 
+function summarizeText(value, limit = TOOL_GATEWAY_SUMMARY_LIMIT) {
+  const text = String(value ?? '');
+  if (text.length <= limit) return text;
+  return text.slice(text.length - limit);
+}
+
+function normalizeExpectedFileCheck(value) {
+  if (typeof value === 'string') {
+    const expectedPath = value.trim();
+    return expectedPath ? { path: expectedPath } : null;
+  }
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const expectedPath = String(value.path ?? value.file ?? value.relativePath ?? '').trim();
+  if (!expectedPath) return null;
+  const contains = value.contains ?? value.expectedContent ?? value.contentIncludes;
+  return {
+    path: expectedPath,
+    ...(contains == null ? {} : { contains: String(contains) }),
+  };
+}
+
+function normalizeToolGatewayValidation(value) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  const checks = [];
+  const single = normalizeExpectedFileCheck(value.expectedFile ?? value.expectedPath);
+  if (single) checks.push(single);
+  if (Array.isArray(value.expectedFiles)) {
+    for (const item of value.expectedFiles) {
+      const check = normalizeExpectedFileCheck(item);
+      if (check) checks.push(check);
+    }
+  }
+  return checks.length > 0 ? { expectedFiles: checks } : null;
+}
+
+function resolveValidationPath(cwd, expectedPath) {
+  if (!String(cwd ?? '').trim()) {
+    const err = new Error('Tool Gateway validation failed: missing working directory');
+    err.code = 'TOOL_GATEWAY_VALIDATION_FAILED';
+    err.retryable = false;
+    throw err;
+  }
+  const root = path.resolve(String(cwd ?? ''));
+  const target = path.resolve(root, String(expectedPath ?? ''));
+  if (target !== root && !target.startsWith(`${root}${path.sep}`)) {
+    const err = new Error(`Tool Gateway validation path escapes working directory: ${expectedPath}`);
+    err.code = 'TOOL_GATEWAY_VALIDATION_FAILED';
+    err.retryable = false;
+    throw err;
+  }
+  return { root, target };
+}
+
+async function validateToolGatewayResult({ result, validation, cwd }) {
+  if (!validation?.expectedFiles?.length || result?.dryRun) {
+    return null;
+  }
+  const checks = [];
+  for (const expected of validation.expectedFiles) {
+    const { root, target } = resolveValidationPath(cwd ?? result?.cwd, expected.path);
+    let content = '';
+    let stat = null;
+    try {
+      stat = await fs.stat(target);
+      content = await fs.readFile(target, 'utf8');
+    } catch (cause) {
+      const err = new Error(`Tool Gateway validation failed: expected file not found: ${expected.path}`);
+      err.code = 'TOOL_GATEWAY_VALIDATION_FAILED';
+      err.retryable = false;
+      err.validation = {
+        expectedFile: expected.path,
+        cwd: root,
+        reason: 'missing_file',
+      };
+      err.cause = cause;
+      throw err;
+    }
+    if (expected.contains != null && !content.includes(expected.contains)) {
+      const err = new Error(`Tool Gateway validation failed: expected content not found in ${expected.path}`);
+      err.code = 'TOOL_GATEWAY_VALIDATION_FAILED';
+      err.retryable = false;
+      err.validation = {
+        expectedFile: expected.path,
+        cwd: root,
+        reason: 'missing_content',
+      };
+      throw err;
+    }
+    checks.push({
+      path: expected.path,
+      sizeBytes: Number(stat?.size ?? 0),
+      contains: expected.contains == null ? null : true,
+    });
+  }
+  return { expectedFiles: checks };
+}
+
 function withRunMetadata(userMessage, { toolMode = 'chat', taskType = null } = {}) {
   const message = (userMessage && typeof userMessage === 'object' && !Array.isArray(userMessage))
     ? { ...userMessage }
@@ -82,6 +182,7 @@ function getRunOptionsFromMessage(userMessage) {
   return {
     toolMode,
     taskType: normalizeTaskType(runMetadata?.taskType ?? metadata?.taskType),
+    validation: normalizeToolGatewayValidation(runMetadata?.validation ?? metadata?.toolGatewayValidation),
   };
 }
 
@@ -275,7 +376,26 @@ export function createAgentRunGateway({
         executor: result.executor ?? null,
         dryRun: Boolean(result.dryRun),
         exitCode: result.exitCode ?? null,
+        stdoutTail: summarizeText(result.stdout),
+        stderrTail: summarizeText(result.stderr),
       });
+      try {
+        const validation = await validateToolGatewayResult({
+          result,
+          validation: runOptions.validation,
+          cwd: workingDir,
+        });
+        if (validation) {
+          await appendEvent(runId, 'tool_gateway_validation', validation);
+        }
+      } catch (err) {
+        await appendEvent(runId, 'tool_gateway_validation_failed', {
+          code: err?.code ?? null,
+          message: err instanceof Error ? err.message : String(err),
+          validation: err?.validation ?? null,
+        });
+        throw err;
+      }
       return { sessionId: row.agent_session_id ?? null };
     }
 
@@ -333,7 +453,7 @@ export function createAgentRunGateway({
       if (timedOut) {
         await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs });
       }
-      const retryable = !timedOut && nextAttempt < retryDelaysMs.length;
+      const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length;
       await markRun(runId, retryable ? 'retryable' : 'failed', {
         error_message: message,
         completed_at: retryable ? null : nowMs(),

agent-run-gateway.test.mjs

@@ -1,4 +1,7 @@
 import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
 import test from 'node:test';
 import { createAgentRunGateway } from './agent-run-gateway.mjs';
 
@@ -276,6 +279,140 @@ test('agent run with enabled tool gateway dispatches code runs outside goose ses
   );
 });
 
+test('agent run validates expected tool gateway artifacts before succeeding', async () => {
+  const pool = createFakePool();
+  const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-'));
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {
+      async resolveWorkingDir() {
+        return workdir;
+      },
+    },
+    tkmindProxy: {
+      async startSessionForUser() {
+        assert.fail('goose session should not start for external tool gateway run');
+      },
+      async submitSessionReplyForUser() {
+        assert.fail('goose reply should not be submitted for external tool gateway run');
+      },
+    },
+    toolGateway: {
+      getStatus() {
+        return { enabled: true, protocol: 'agent-run-v1' };
+      },
+      async executeJob() {
+        await fs.writeFile(
+          path.join(workdir, 'RESULT.md'),
+          'validated artifact from tool gateway\n',
+          'utf8',
+        );
+        return {
+          ok: true,
+          dryRun: false,
+          executor: 'aider',
+          exitCode: 0,
+          cwd: workdir,
+          stdout: 'created RESULT.md',
+          stderr: '',
+        };
+      },
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-code-tool-validation',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: 'create validation artifact' }],
+      metadata: {
+        memindRun: {
+          validation: {
+            expectedFile: {
+              path: 'RESULT.md',
+              contains: 'validated artifact',
+            },
+          },
+        },
+      },
+    },
+    toolMode: 'code',
+    taskType: 'small_patch',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  const validationEvent = pool.events.find(
+    (event) => event.runId === run.id && event.eventType === 'tool_gateway_validation',
+  );
+  assert.ok(validationEvent);
+  assert.equal(JSON.parse(validationEvent.dataJson).expectedFiles[0].path, 'RESULT.md');
+});
+
+test('agent run fails non-retryably when tool gateway artifact validation fails', async () => {
+  const pool = createFakePool();
+  const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-missing-'));
+  let attempts = 0;
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {
+      async resolveWorkingDir() {
+        return workdir;
+      },
+    },
+    tkmindProxy: {
+      async startSessionForUser() {
+        assert.fail('goose session should not start for external tool gateway run');
+      },
+      async submitSessionReplyForUser() {
+        assert.fail('goose reply should not be submitted for external tool gateway run');
+      },
+    },
+    toolGateway: {
+      getStatus() {
+        return { enabled: true, protocol: 'agent-run-v1' };
+      },
+      async executeJob() {
+        attempts += 1;
+        return {
+          ok: true,
+          dryRun: false,
+          executor: 'aider',
+          exitCode: 0,
+          cwd: workdir,
+        };
+      },
+    },
+    retryDelaysMs: [0, 0],
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-code-tool-validation-fail',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: 'forget to create validation artifact' }],
+      metadata: {
+        memindRun: {
+          validation: {
+            expectedFile: 'MISSING.md',
+          },
+        },
+      },
+    },
+    toolMode: 'code',
+    taskType: 'small_patch',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
+  assert.equal(attempts, 1);
+  assert.equal(pool.runs.get(run.id).attempts, 1);
+  assert.match(pool.runs.get(run.id).error_message, /expected file not found/);
+  assert.equal(
+    pool.events.some((event) => event.runId === run.id && event.eventType === 'tool_gateway_validation_failed'),
+    true,
+  );
+});
+
 test('agent run retries transient failures and then becomes terminal', async () => {
   const pool = createFakePool();
   const gateway = createAgentRunGateway({