From 7a47dc6a987cef71badee76d7d7f2047d41df5c2 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 12:31:29 +0800 Subject: [PATCH] feat: add task artifact validation for code runs --- ...2-runtime-execution-assessment-20260702.md | 41 ++++- .../memind-2-streaming-agent-runtime-plan.md | 66 ++++++++ src/hooks/usePageEditSubChat.ts | 10 +- src/hooks/useTKMindChat.ts | 6 +- src/utils/agentRunMode.ts | 142 +++++++++++++++++- 5 files changed, 251 insertions(+), 14 deletions(-) diff --git a/docs/architecture/memind-2-runtime-execution-assessment-20260702.md b/docs/architecture/memind-2-runtime-execution-assessment-20260702.md index 9a5cf27..fd2bd67 100644 --- a/docs/architecture/memind-2-runtime-execution-assessment-20260702.md +++ b/docs/architecture/memind-2-runtime-execution-assessment-20260702.md @@ -19,12 +19,12 @@ - Stream Controller 已具备 SSE headers、abort propagation、backpressure pipeline。 - Redis Router 已启用，承担 worker runtime state。 - Goose Worker Pool 已从固定单点走向四 worker 可观测调度。 -- Aider/OpenHands 已从普通聊天默认能力中剥离，进入 code mode 和后端灰度门禁；P6.3 已把 code run 从 goosed session extension 外移到 `agent-run-v1` Tool Gateway 协议，P6.4 已完成 Aider 真实执行 canary，P6.5 已完成 OpenHands 真实执行 canary，P6.6 已完成 external worker 精确接管 code-run canary，P6.7 已加入 Tool Gateway 产物校验与输出审计，P6.8 已安装 external worker LaunchAgent，P6.9 已完成带 validation 的 external worker 灰度 canary，P6.10 已加入 external worker 只读观测脚本，P6.11 已加入放量策略门禁，P6.12 已开启全用户长期灰度并通过普通测试用户真实路径，P6.13 已安装自动暂停 guard，P6.15 已让 H5 code-run 自动补 receipt validation 并恢复后端 required validation，P6.17/P6.18 已把 external worker 并发 2 通过 canary 并固化为当前 all-user gray 策略，P8.1 已加入 stale running recovery，P8.2 已完成 Portal DB/Auth 瞬时错误兜底。 +- Aider/OpenHands 已从普通聊天默认能力中剥离，进入 code mode 和后端灰度门禁；P6.3 已把 code run 从 goosed session extension 外移到 `agent-run-v1` Tool Gateway 协议，P6.4 已完成 Aider 真实执行 canary，P6.5 已完成 OpenHands 真实执行 canary，P6.6 已完成 external worker 精确接管 code-run canary，P6.7 已加入 Tool Gateway 产物校验与输出审计，P6.8 已安装 external worker LaunchAgent，P6.9 已完成带 validation 的 external worker 灰度 canary，P6.10 已加入 external worker 只读观测脚本，P6.11 已加入放量策略门禁，P6.12 已开启全用户长期灰度并通过普通测试用户真实路径，P6.13 已安装自动暂停 guard，P6.15 已让 H5 code-run 自动补 receipt validation 并恢复后端 required validation，P6.17/P6.18 已把 external worker 并发 2 通过 canary 并固化为当前 all-user gray 策略，P6.19 已加入 task-level artifact validation，P8.1 已加入 stale running recovery，P8.2 已完成 Portal DB/Auth 瞬时错误兜底。 - PG 和 MindSpace 仍保持生产数据边界，SLO 报告只做统计读取；P6.3-P6.15 不新增 schema migration，不删除或修改既有用户数据。 整体执行评分: 9.95 / 10。 -可以支撑当前 H5 streaming 稳定性改造的基础目标。自动采样、worker sidecar heartbeat、SLO 只读快照、SLO 日报定时器、SLO 日报保留策略、first-token EWMA、first-token p50/p95 窗口趋势、Tool Gateway Queue v0、外部 worker 接管入口、真实 worker canary、后端 code-mode canary、code-run 用户级灰度 gate、H5 页面编辑 UI canary、P6.3 Tool Gateway 协议化、P6.4 Aider 真实 canary、P6.5 OpenHands 真实 canary、P6.6 external worker code-run canary、P6.7 Tool Gateway guardrails、P6.8 worker LaunchAgent、P6.9 validated external worker canary、P6.10 worker observability、P6.11 rollout policy gates、P6.12 all-user gray、P6.13 auto-pause guard、P6.15 H5 validation metadata、P6.17/P6.18 worker concurrency 2、P8.1 stale running recovery 和 P8.2 Portal DB/Auth transient hardening 已经落地。主要剩余差距转为更细粒度的任务级产物规范和用户可见进度/失败说明。 +可以支撑当前 H5 streaming 稳定性改造的基础目标。自动采样、worker sidecar heartbeat、SLO 只读快照、SLO 日报定时器、SLO 日报保留策略、first-token EWMA、first-token p50/p95 窗口趋势、Tool Gateway Queue v0、外部 worker 接管入口、真实 worker canary、后端 code-mode canary、code-run 用户级灰度 gate、H5 页面编辑 UI canary、P6.3 Tool Gateway 协议化、P6.4 Aider 真实 canary、P6.5 OpenHands 真实 canary、P6.6 external worker code-run canary、P6.7 Tool Gateway guardrails、P6.8 worker LaunchAgent、P6.9 validated external worker canary、P6.10 worker observability、P6.11 rollout policy gates、P6.12 all-user gray、P6.13 auto-pause guard、P6.15 H5 validation metadata、P6.17/P6.18 worker concurrency 2、P6.19 task-level artifact validation、P8.1 stale running recovery 和 P8.2 Portal DB/Auth transient hardening 已经落地。主要剩余差距转为用户可见进度/失败说明，以及后续更细任务类型的校验策略扩展。 ## 实测结果 @@ -941,13 +941,40 @@ Data boundary: - guard `shouldPause=false` - SLO `ok=true`, `failures=[]` -## 下一步执行建议 - ### P6.19 Task-level Artifact Validation -- receipt 继续作为兜底。 -- 页面编辑任务追加目标 `public/*.html` 校验。 -- 文件/仓库任务根据任务参数声明目标相对路径。 +结果: 通过，H5 默认 code-run 在 receipt 之外追加 task-level expectedFiles，生产真实 canary 已验证三项产物校验同时通过。 + +- receipt 继续作为兜底: + - `.memind/agent-runs/.json` +- 普通 H5 code-run: + - 从用户输入、MindSpace 上下文和公开链接中提取 `public/*.html`。 + - 将可推断的目标 HTML 文件追加到 `metadata.memindRun.validation.expectedFiles`。 +- 页面编辑 code-run: + - 传入 `pageId`、`pageTitle` 和 MindSpace context。 + - 追加 `.memind/agent-runs/-page-edit.json`，内容必须包含 `pageId`。 +- 测试: + - `npm run build` + - `node --test agent-run-gateway.test.mjs agent-run-routes.test.mjs`，29 tests pass。 + - `npx tsc --noEmit` 仍被既有历史 TS 问题阻断，本次相关返回类型已收紧。 +- 生产部署: + - backup `/Users/john/Project/memind_backups/20260702-122737-p619-task-artifact-validation` + - H5 bundle `/assets/index-jYf9W5vk.js` +- real canary: + - run `60067d2f-d2af-40d1-bf35-7976309bd8da` + - request `p619-task-validation-20260702042911` + - status `succeeded` + - event `tool_gateway_validation` + - expectedFiles: + - `.memind/agent-runs/p619-task-validation-20260702042911.json` + - `public/p619-task-validation-20260702042911.html` + - `.memind/agent-runs/p619-task-validation-20260702042911-page-edit.json` +- post-check: + - runtime/status all-user gray unchanged + - guard `shouldPause=false` + - `failedRecentCount=0` + +## 下一步执行建议 ### P6.22 Task Artifact UX and Failure Messages diff --git a/docs/architecture/memind-2-streaming-agent-runtime-plan.md b/docs/architecture/memind-2-streaming-agent-runtime-plan.md index c562149..a19c610 100644 --- a/docs/architecture/memind-2-streaming-agent-runtime-plan.md +++ b/docs/architecture/memind-2-streaming-agent-runtime-plan.md @@ -26,6 +26,7 @@ - P6.15 H5 Code-run Validation Metadata: 已完成，H5 code-run 自动声明 `.memind/agent-runs/.json` receipt 校验，生产已恢复 `MEMIND_AGENT_CODE_RUNS_REQUIRE_VALIDATION=1` 并通过普通测试用户真实路径。 - P6.17 Controlled Worker Concurrency Canary: 已完成，external worker 短窗口提升到并发 2，两条真实 code-run 同时 running 并全部通过 receipt validation。 - P6.18 Concurrency Rollout Policy: 已完成，生产 worker 和 Portal runtime/status 已对齐到并发 2，worker KeepAlive 保持开启，保留回滚备份。 +- P6.19 Task-level Artifact Validation: 已完成第一步，H5 code-run 会在 receipt 之外追加可推断的 `public/*.html` 目标文件校验，页面编辑 code-run 会追加 page-edit task receipt，生产真实 canary 三项 expectedFiles 全部通过。 - P8.1 Queue Lease / Stuck Run Recovery: 已完成第一步，worker dispatch 前自动回收超时 running run，新增 `--recover-stale` dry-run/apply 运维入口，生产 synthetic stale run 验证通过。 - P8.2 Portal DB/Auth Transient Error Hardening: 已完成第一步，session attach、`/auth/status` 和 API auth middleware 已捕获 DB/auth 瞬时错误，生产部署后 live health、auth/status、runtime/status、guard 和 SLO 验证通过。 - P5.15 Active Stream TTL Reconcile: 已按用户要求跳过，暂不做报表/定时 reconcile。 @@ -2863,6 +2864,71 @@ runtime/status: - `ok=true` - `failures=[]` +### 2026-07-02 P6.19 Task-level Artifact Validation + +目标: + +- receipt validation 继续作为所有 code-run 的兜底。 +- 对能从用户指令或 MindSpace 上下文稳定推断的具体产物追加 task-level validation。 +- 页面编辑类任务即使通过 API/patch 修改页面，也需要生成带 `pageId` 的 task receipt，证明执行器处理的是目标页面。 + +改动: + +- `src/utils/agentRunMode.ts`: + - 新增 `extractAgentRunPublicHtmlPaths()`，可从用户输入、`MindSpace/.../public/*.html` 和公开链接中提取 workspace-relative `public/*.html`。 + - 新增 `buildAgentRunTaskValidation()`。 + - `resolveAgentRunOptions()` 现在会合并: + - 默认 `.memind/agent-runs/.json` receipt。 + - 推断出的 `public/*.html` expectedFiles。 + - `page_edit_code_task` 的 `.memind/agent-runs/-page-edit.json`，内容必须包含 `pageId`。 +- `src/hooks/useTKMindChat.ts`: + - 普通 H5 code-run 调用 `resolveAgentRunOptions()` 时传入 MindSpace context。 +- `src/hooks/usePageEditSubChat.ts`: + - 页面编辑子聊天传入 MindSpace context、`pageId` 和 `pageTitle`。 + +测试: + +- `npm run build` 通过。 +- `node --test agent-run-gateway.test.mjs agent-run-routes.test.mjs` 通过，29 tests pass。 +- `npx tsc --noEmit` 仍被既有历史 TS 问题阻断，本次相关 `waitForAgentRun()` 返回类型已收紧；最终以 Vite build 和 agent-run route/gateway tests 为准。 + +生产部署: + +- 使用灰度 env 重建 H5: + - `VITE_AGENT_CODE_RUNS_ENABLED=1` + - `VITE_AGENT_CODE_RUNS_AUTODETECT=1` + - `VITE_AGENT_CODE_RUNS_USER_IDS=` +- 备份: + - `/Users/john/Project/memind_backups/20260702-122737-p619-task-artifact-validation` +- 已同步: + - `/Users/john/Project/Memind/dist` + +生产验证: + +- `https://mm.tkmind.cn/` 已加载新 bundle: + - `/assets/index-jYf9W5vk.js` +- `/api/runtime/status`: + - `ok=true` + - `codeRunsEnabled=true` + - `requireValidation=true` + - `maxConcurrentRuns=2` + - `statusCounts={}` +- guard dry-run: + - `ok=true` + - `shouldPause=false` + - `failedRecentCount=0` +- real canary: + - user `916f1fe6-e9c8-44b5-9faa-2fc1bf146ab9` + - run `60067d2f-d2af-40d1-bf35-7976309bd8da` + - request `p619-task-validation-20260702042911` + - status `succeeded` + - attempts `1` + - event chain includes `tool_gateway_validation` + - validated expectedFiles: + - `.memind/agent-runs/p619-task-validation-20260702042911.json` + - `public/p619-task-validation-20260702042911.html` + - `.memind/agent-runs/p619-task-validation-20260702042911-page-edit.json` + ## 回滚策略 - P0: 修改前保留 `server.mjs` 备份；如启动失败，恢复备份并 `launchctl kickstart` Portal。 diff --git a/src/hooks/usePageEditSubChat.ts b/src/hooks/usePageEditSubChat.ts index 6476d40..8564c8c 100644 --- a/src/hooks/usePageEditSubChat.ts +++ b/src/hooks/usePageEditSubChat.ts @@ -13,6 +13,7 @@ import { uploadMindSpaceAsset, subscribeSessionEvents, } from '../api/client'; +import type { AgentRun } from '../api/client'; import type { ChatState, Message, MindSpaceChatContext, PortalUser, Session, SessionEvent } from '../types'; import { buildContextPrefix } from '../utils/mindspaceChatContext'; import { buildUserAddressPrefix } from '../utils/userAddress'; @@ -29,8 +30,8 @@ import { pushMessage, } from '../utils/message'; -async function waitForAgentRun(runId: string) { - return await new Promise((resolve, reject) => { +async function waitForAgentRun(runId: string): Promise { + return await new Promise((resolve, reject) => { const unsubscribe = subscribeAgentRunEvents( runId, (run) => { @@ -349,6 +350,11 @@ export function usePageEditSubChat({ forceCode: true, userId: user?.id ?? null, requestId, + mindspaceContext: context, + pageEdit: { + pageId, + pageTitle, + }, }), ); const finishedRun = diff --git a/src/hooks/useTKMindChat.ts b/src/hooks/useTKMindChat.ts index da0ccac..f324328 100644 --- a/src/hooks/useTKMindChat.ts +++ b/src/hooks/useTKMindChat.ts @@ -24,6 +24,7 @@ import { syncUserMemory, updateProvider, } from '../api/client'; +import type { AgentRun } from '../api/client'; import { appConfig } from '../config'; import { clearStoredSessionId, @@ -74,8 +75,8 @@ const FINISH_SYNC_RETRY_DELAYS_MS = [500, 1500, 3000]; const ACTIVE_REQUEST_MISSING_GRACE_MS = 2500; export { INSUFFICIENT_BALANCE_NOTICE }; -async function waitForAgentRun(runId: string) { - return await new Promise((resolve, reject) => { +async function waitForAgentRun(runId: string): Promise { + return await new Promise((resolve, reject) => { const unsubscribe = subscribeAgentRunEvents( runId, (run) => { @@ -1177,6 +1178,7 @@ export function useTKMindChat( taskType: 'h5_chat_code_task', userId: userRef.current?.id ?? null, requestId, + mindspaceContext: options?.mindspaceContext ?? null, }), ); const finishedRun = diff --git a/src/utils/agentRunMode.ts b/src/utils/agentRunMode.ts index 037d3cf..24dd4d1 100644 --- a/src/utils/agentRunMode.ts +++ b/src/utils/agentRunMode.ts @@ -1,3 +1,5 @@ +import type { MindSpaceChatContext } from '../types'; + export type AgentRunCreateOptions = { toolMode?: 'chat' | 'code'; taskType?: string | null; @@ -55,6 +57,128 @@ function sanitizeRequestIdForPath(requestId: string): string { return normalized || 'unknown-request'; } +function normalizeExpectedFile(value: AgentRunValidationFile | string): AgentRunValidationFile | null { + if (typeof value === 'string') { + const path = value.trim(); + return path ? { path } : null; + } + const path = String(value?.path ?? '').trim(); + if (!path) return null; + return { + path, + ...(value.contains == null ? {} : { contains: String(value.contains) }), + }; +} + +function mergeAgentRunValidationFiles( + ...items: Array +): AgentRunValidation | null { + const byKey = new Map(); + const addFile = (file: AgentRunValidationFile | string | null | undefined) => { + if (!file) return; + const normalized = normalizeExpectedFile(file); + if (!normalized) return; + const key = `${normalized.path}\u0000${normalized.contains ?? ''}`; + byKey.set(key, normalized); + }; + + for (const item of items) { + if (!item) continue; + if (typeof item === 'string' || 'path' in item) { + addFile(item); + continue; + } + addFile(item.expectedFile); + for (const file of item.expectedFiles ?? []) addFile(file); + } + + const expectedFiles = [...byKey.values()]; + return expectedFiles.length ? { expectedFiles } : null; +} + +export function normalizeAgentRunPublicHtmlPath(candidate: string): string | null { + const decoded = decodeURIComponent(String(candidate ?? '')).replace(/\\/g, '/').trim(); + const publicMatch = decoded.match(/(?:^|\/)(public\/[^"'<>?#\s]+\.html)\b/i); + const raw = publicMatch?.[1] ?? ''; + if (!raw) return null; + const parts = raw + .split('/') + .map((part) => part.trim()) + .filter(Boolean); + if (parts.length < 2 || parts[0] !== 'public') return null; + if (parts.some((part) => part === '.' || part === '..')) return null; + return parts.join('/'); +} + +export function extractAgentRunPublicHtmlPaths(...values: Array): string[] { + const found = new Set(); + const pattern = /(?:^|[\s"'([{<])((?:https?:\/\/[^\s"'<>]+|\/?MindSpace\/[^\s"'<>]+|public\/[^\s"'<>]+)\.html)\b/gi; + for (const value of values) { + const text = String(value ?? ''); + let match: RegExpExecArray | null; + while ((match = pattern.exec(text))) { + const path = normalizeAgentRunPublicHtmlPath(match[1] ?? ''); + if (path) found.add(path); + } + } + return [...found]; +} + +export function buildAgentRunTaskValidation({ + requestId, + taskType, + text, + mindspaceContext, + pageEdit, +}: { + requestId: string; + taskType: string; + text: string; + mindspaceContext?: MindSpaceChatContext | null; + pageEdit?: { pageId?: string | null; pageTitle?: string | null } | null; +}): { validation: AgentRunValidation | null; instruction: string } { + const safeRequestId = sanitizeRequestIdForPath(requestId); + const pageId = String(pageEdit?.pageId ?? mindspaceContext?.page?.id ?? '').trim(); + const pageTitle = String(pageEdit?.pageTitle ?? mindspaceContext?.page?.title ?? '').trim(); + const publicHtmlPaths = extractAgentRunPublicHtmlPaths( + text, + mindspaceContext?.page?.publicationUrl ?? null, + ); + const expectedFiles: AgentRunValidationFile[] = publicHtmlPaths.map((path) => ({ path })); + const instructions: string[] = []; + + if (publicHtmlPaths.length > 0) { + instructions.push( + '', + '[Memind task artifact validation]', + 'The task references concrete MindSpace public HTML artifacts.', + 'Before finishing, ensure these workspace-relative files exist and contain the intended result:', + ...publicHtmlPaths.map((path) => `- ${path}`), + ); + } + + if (taskType === 'page_edit_code_task' && pageId) { + const taskReceiptPath = `.memind/agent-runs/${safeRequestId}-page-edit.json`; + expectedFiles.push({ path: taskReceiptPath, contains: pageId }); + instructions.push( + '', + '[Memind page-edit validation]', + `Before finishing this page-edit task, create or update ${taskReceiptPath}.`, + 'The file must be valid JSON and include:', + `- requestId: ${requestId}`, + `- taskType: ${taskType}`, + `- pageId: ${pageId}`, + ...(pageTitle ? [`- pageTitle: ${pageTitle}`] : []), + '- a brief summary of the page changes or the reason no change was needed.', + ); + } + + return { + validation: expectedFiles.length ? { expectedFiles } : null, + instruction: instructions.join('\n'), + }; +} + export function buildAgentRunValidationReceipt(requestId: string): { validation: AgentRunValidation; instruction: string; @@ -88,23 +212,35 @@ export function resolveAgentRunOptions( allowAutodetect = agentCodeRunsAutodetectEnabled, userId = null, requestId = null, + mindspaceContext = null, + pageEdit = null, }: { taskType?: string; forceCode?: boolean; allowAutodetect?: boolean; userId?: string | null; requestId?: string | null; + mindspaceContext?: MindSpaceChatContext | null; + pageEdit?: { pageId?: string | null; pageTitle?: string | null } | null; } = {}, ): AgentRunCreateOptions { if (!agentCodeRunsEnabledForUser(userId)) return {}; const normalizedText = String(text ?? '').trim(); const shouldUseCode = forceCode || (allowAutodetect && CODE_TASK_PATTERNS.some((pattern) => pattern.test(normalizedText))); if (!shouldUseCode) return {}; - const receipt = buildAgentRunValidationReceipt(requestId ?? crypto.randomUUID()); + const normalizedRequestId = requestId ?? crypto.randomUUID(); + const receipt = buildAgentRunValidationReceipt(normalizedRequestId); + const taskValidation = buildAgentRunTaskValidation({ + requestId: normalizedRequestId, + taskType, + text: normalizedText, + mindspaceContext, + pageEdit, + }); return { toolMode: 'code', taskType, - validation: receipt.validation, - validationInstruction: receipt.instruction, + validation: mergeAgentRunValidationFiles(receipt.validation, taskValidation.validation), + validationInstruction: `${receipt.instruction}${taskValidation.instruction}`, }; }