feat: validate tool gateway artifacts

2026-07-02 10:12:30 +08:00
parent a2d69a0c36
commit 6340b57fda
2 changed files with 258 additions and 1 deletions
@@ -1,4 +1,7 @@
 import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
 import test from 'node:test';
 import { createAgentRunGateway } from './agent-run-gateway.mjs';

@@ -276,6 +279,140 @@ test('agent run with enabled tool gateway dispatches code runs outside goose ses
  );
 });

+test('agent run validates expected tool gateway artifacts before succeeding', async () => {
+  const pool = createFakePool();
+  const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-'));
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {
+      async resolveWorkingDir() {
+        return workdir;
+      },
+    },
+    tkmindProxy: {
+      async startSessionForUser() {
+        assert.fail('goose session should not start for external tool gateway run');
+      },
+      async submitSessionReplyForUser() {
+        assert.fail('goose reply should not be submitted for external tool gateway run');
+      },
+    },
+    toolGateway: {
+      getStatus() {
+        return { enabled: true, protocol: 'agent-run-v1' };
+      },
+      async executeJob() {
+        await fs.writeFile(
+          path.join(workdir, 'RESULT.md'),
+          'validated artifact from tool gateway\n',
+          'utf8',
+        );
+        return {
+          ok: true,
+          dryRun: false,
+          executor: 'aider',
+          exitCode: 0,
+          cwd: workdir,
+          stdout: 'created RESULT.md',
+          stderr: '',
+        };
+      },
+    },
+    retryDelaysMs: [],
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-code-tool-validation',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: 'create validation artifact' }],
+      metadata: {
+        memindRun: {
+          validation: {
+            expectedFile: {
+              path: 'RESULT.md',
+              contains: 'validated artifact',
+            },
+          },
+        },
+      },
+    },
+    toolMode: 'code',
+    taskType: 'small_patch',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
+  const validationEvent = pool.events.find(
+    (event) => event.runId === run.id && event.eventType === 'tool_gateway_validation',
+  );
+  assert.ok(validationEvent);
+  assert.equal(JSON.parse(validationEvent.dataJson).expectedFiles[0].path, 'RESULT.md');
+});
+
+test('agent run fails non-retryably when tool gateway artifact validation fails', async () => {
+  const pool = createFakePool();
+  const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-missing-'));
+  let attempts = 0;
+  const gateway = createAgentRunGateway({
+    pool,
+    userAuth: {
+      async resolveWorkingDir() {
+        return workdir;
+      },
+    },
+    tkmindProxy: {
+      async startSessionForUser() {
+        assert.fail('goose session should not start for external tool gateway run');
+      },
+      async submitSessionReplyForUser() {
+        assert.fail('goose reply should not be submitted for external tool gateway run');
+      },
+    },
+    toolGateway: {
+      getStatus() {
+        return { enabled: true, protocol: 'agent-run-v1' };
+      },
+      async executeJob() {
+        attempts += 1;
+        return {
+          ok: true,
+          dryRun: false,
+          executor: 'aider',
+          exitCode: 0,
+          cwd: workdir,
+        };
+      },
+    },
+    retryDelaysMs: [0, 0],
+  });
+
+  const run = await gateway.createRun('user-1', {
+    requestId: 'req-code-tool-validation-fail',
+    userMessage: {
+      role: 'user',
+      content: [{ type: 'text', text: 'forget to create validation artifact' }],
+      metadata: {
+        memindRun: {
+          validation: {
+            expectedFile: 'MISSING.md',
+          },
+        },
+      },
+    },
+    toolMode: 'code',
+    taskType: 'small_patch',
+  });
+
+  await waitFor(() => pool.runs.get(run.id)?.status === 'failed');
+  assert.equal(attempts, 1);
+  assert.equal(pool.runs.get(run.id).attempts, 1);
+  assert.match(pool.runs.get(run.id).error_message, /expected file not found/);
+  assert.equal(
+    pool.events.some((event) => event.runId === run.id && event.eventType === 'tool_gateway_validation_failed'),
+    true,
+  );
+});
+
 test('agent run retries transient failures and then becomes terminal', async () => {
  const pool = createFakePool();
  const gateway = createAgentRunGateway({