diff --git a/agent-run-gateway.mjs b/agent-run-gateway.mjs index 3729763..e7ebd70 100644 --- a/agent-run-gateway.mjs +++ b/agent-run-gateway.mjs @@ -1,4 +1,6 @@ import crypto from 'node:crypto'; +import fs from 'node:fs/promises'; +import path from 'node:path'; const DEFAULT_RUN_RETRY_DELAYS_MS = [1500, 5000, 15000]; const TERMINAL_STATUSES = new Set(['succeeded', 'failed']); @@ -6,6 +8,7 @@ const CODE_TOOL_MODES = new Set(['code', 'code-task', 'code_task', 'code-tool', const RUN_METADATA_KEY = 'memindRun'; const DEFAULT_MAX_CONCURRENT_RUNS = 1; const DEFAULT_RUN_TIMEOUT_MS = 15 * 60 * 1000; +const TOOL_GATEWAY_SUMMARY_LIMIT = 4096; function nowMs() { return Date.now(); @@ -53,6 +56,103 @@ function normalizeTaskType(value) { return normalized || null; } +function summarizeText(value, limit = TOOL_GATEWAY_SUMMARY_LIMIT) { + const text = String(value ?? ''); + if (text.length <= limit) return text; + return text.slice(text.length - limit); +} + +function normalizeExpectedFileCheck(value) { + if (typeof value === 'string') { + const expectedPath = value.trim(); + return expectedPath ? { path: expectedPath } : null; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const expectedPath = String(value.path ?? value.file ?? value.relativePath ?? '').trim(); + if (!expectedPath) return null; + const contains = value.contains ?? value.expectedContent ?? value.contentIncludes; + return { + path: expectedPath, + ...(contains == null ? {} : { contains: String(contains) }), + }; +} + +function normalizeToolGatewayValidation(value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return null; + const checks = []; + const single = normalizeExpectedFileCheck(value.expectedFile ?? value.expectedPath); + if (single) checks.push(single); + if (Array.isArray(value.expectedFiles)) { + for (const item of value.expectedFiles) { + const check = normalizeExpectedFileCheck(item); + if (check) checks.push(check); + } + } + return checks.length > 0 ? { expectedFiles: checks } : null; +} + +function resolveValidationPath(cwd, expectedPath) { + if (!String(cwd ?? '').trim()) { + const err = new Error('Tool Gateway validation failed: missing working directory'); + err.code = 'TOOL_GATEWAY_VALIDATION_FAILED'; + err.retryable = false; + throw err; + } + const root = path.resolve(String(cwd ?? '')); + const target = path.resolve(root, String(expectedPath ?? '')); + if (target !== root && !target.startsWith(`${root}${path.sep}`)) { + const err = new Error(`Tool Gateway validation path escapes working directory: ${expectedPath}`); + err.code = 'TOOL_GATEWAY_VALIDATION_FAILED'; + err.retryable = false; + throw err; + } + return { root, target }; +} + +async function validateToolGatewayResult({ result, validation, cwd }) { + if (!validation?.expectedFiles?.length || result?.dryRun) { + return null; + } + const checks = []; + for (const expected of validation.expectedFiles) { + const { root, target } = resolveValidationPath(cwd ?? result?.cwd, expected.path); + let content = ''; + let stat = null; + try { + stat = await fs.stat(target); + content = await fs.readFile(target, 'utf8'); + } catch (cause) { + const err = new Error(`Tool Gateway validation failed: expected file not found: ${expected.path}`); + err.code = 'TOOL_GATEWAY_VALIDATION_FAILED'; + err.retryable = false; + err.validation = { + expectedFile: expected.path, + cwd: root, + reason: 'missing_file', + }; + err.cause = cause; + throw err; + } + if (expected.contains != null && !content.includes(expected.contains)) { + const err = new Error(`Tool Gateway validation failed: expected content not found in ${expected.path}`); + err.code = 'TOOL_GATEWAY_VALIDATION_FAILED'; + err.retryable = false; + err.validation = { + expectedFile: expected.path, + cwd: root, + reason: 'missing_content', + }; + throw err; + } + checks.push({ + path: expected.path, + sizeBytes: Number(stat?.size ?? 0), + contains: expected.contains == null ? null : true, + }); + } + return { expectedFiles: checks }; +} + function withRunMetadata(userMessage, { toolMode = 'chat', taskType = null } = {}) { const message = (userMessage && typeof userMessage === 'object' && !Array.isArray(userMessage)) ? { ...userMessage } @@ -82,6 +182,7 @@ function getRunOptionsFromMessage(userMessage) { return { toolMode, taskType: normalizeTaskType(runMetadata?.taskType ?? metadata?.taskType), + validation: normalizeToolGatewayValidation(runMetadata?.validation ?? metadata?.toolGatewayValidation), }; } @@ -275,7 +376,26 @@ export function createAgentRunGateway({ executor: result.executor ?? null, dryRun: Boolean(result.dryRun), exitCode: result.exitCode ?? null, + stdoutTail: summarizeText(result.stdout), + stderrTail: summarizeText(result.stderr), }); + try { + const validation = await validateToolGatewayResult({ + result, + validation: runOptions.validation, + cwd: workingDir, + }); + if (validation) { + await appendEvent(runId, 'tool_gateway_validation', validation); + } + } catch (err) { + await appendEvent(runId, 'tool_gateway_validation_failed', { + code: err?.code ?? null, + message: err instanceof Error ? err.message : String(err), + validation: err?.validation ?? null, + }); + throw err; + } return { sessionId: row.agent_session_id ?? null }; } @@ -333,7 +453,7 @@ export function createAgentRunGateway({ if (timedOut) { await appendEvent(runId, 'timeout', { timeoutMs: runTimeoutMs }); } - const retryable = !timedOut && nextAttempt < retryDelaysMs.length; + const retryable = !timedOut && err?.retryable !== false && nextAttempt < retryDelaysMs.length; await markRun(runId, retryable ? 'retryable' : 'failed', { error_message: message, completed_at: retryable ? null : nowMs(), diff --git a/agent-run-gateway.test.mjs b/agent-run-gateway.test.mjs index 5c73fb3..ee9808b 100644 --- a/agent-run-gateway.test.mjs +++ b/agent-run-gateway.test.mjs @@ -1,4 +1,7 @@ import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; import test from 'node:test'; import { createAgentRunGateway } from './agent-run-gateway.mjs'; @@ -276,6 +279,140 @@ test('agent run with enabled tool gateway dispatches code runs outside goose ses ); }); +test('agent run validates expected tool gateway artifacts before succeeding', async () => { + const pool = createFakePool(); + const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-')); + const gateway = createAgentRunGateway({ + pool, + userAuth: { + async resolveWorkingDir() { + return workdir; + }, + }, + tkmindProxy: { + async startSessionForUser() { + assert.fail('goose session should not start for external tool gateway run'); + }, + async submitSessionReplyForUser() { + assert.fail('goose reply should not be submitted for external tool gateway run'); + }, + }, + toolGateway: { + getStatus() { + return { enabled: true, protocol: 'agent-run-v1' }; + }, + async executeJob() { + await fs.writeFile( + path.join(workdir, 'RESULT.md'), + 'validated artifact from tool gateway\n', + 'utf8', + ); + return { + ok: true, + dryRun: false, + executor: 'aider', + exitCode: 0, + cwd: workdir, + stdout: 'created RESULT.md', + stderr: '', + }; + }, + }, + retryDelaysMs: [], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-code-tool-validation', + userMessage: { + role: 'user', + content: [{ type: 'text', text: 'create validation artifact' }], + metadata: { + memindRun: { + validation: { + expectedFile: { + path: 'RESULT.md', + contains: 'validated artifact', + }, + }, + }, + }, + }, + toolMode: 'code', + taskType: 'small_patch', + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded'); + const validationEvent = pool.events.find( + (event) => event.runId === run.id && event.eventType === 'tool_gateway_validation', + ); + assert.ok(validationEvent); + assert.equal(JSON.parse(validationEvent.dataJson).expectedFiles[0].path, 'RESULT.md'); +}); + +test('agent run fails non-retryably when tool gateway artifact validation fails', async () => { + const pool = createFakePool(); + const workdir = await fs.mkdtemp(path.join(os.tmpdir(), 'memind-tool-validation-missing-')); + let attempts = 0; + const gateway = createAgentRunGateway({ + pool, + userAuth: { + async resolveWorkingDir() { + return workdir; + }, + }, + tkmindProxy: { + async startSessionForUser() { + assert.fail('goose session should not start for external tool gateway run'); + }, + async submitSessionReplyForUser() { + assert.fail('goose reply should not be submitted for external tool gateway run'); + }, + }, + toolGateway: { + getStatus() { + return { enabled: true, protocol: 'agent-run-v1' }; + }, + async executeJob() { + attempts += 1; + return { + ok: true, + dryRun: false, + executor: 'aider', + exitCode: 0, + cwd: workdir, + }; + }, + }, + retryDelaysMs: [0, 0], + }); + + const run = await gateway.createRun('user-1', { + requestId: 'req-code-tool-validation-fail', + userMessage: { + role: 'user', + content: [{ type: 'text', text: 'forget to create validation artifact' }], + metadata: { + memindRun: { + validation: { + expectedFile: 'MISSING.md', + }, + }, + }, + }, + toolMode: 'code', + taskType: 'small_patch', + }); + + await waitFor(() => pool.runs.get(run.id)?.status === 'failed'); + assert.equal(attempts, 1); + assert.equal(pool.runs.get(run.id).attempts, 1); + assert.match(pool.runs.get(run.id).error_message, /expected file not found/); + assert.equal( + pool.events.some((event) => event.runId === run.id && event.eventType === 'tool_gateway_validation_failed'), + true, + ); +}); + test('agent run retries transient failures and then becomes terminal', async () => { const pool = createFakePool(); const gateway = createAgentRunGateway({