From 1d1af5c6577b19fd5cd7f1e42fe413c083a5aa38 Mon Sep 17 00:00:00 2001 From: john Date: Wed, 2 Sep 2026 14:52:45 +0800 Subject: [PATCH] Add local script to verify Experience reflect aggregation end-to-end. Seeds task outcomes, runs reflect(), and checks execution_pattern rows plus experience_saved product events for local acceptance. Co-authored-by: Cursor --- package.json | 1 + scripts/verify-experience-reflect-local.mjs | 131 ++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 scripts/verify-experience-reflect-local.mjs diff --git a/package.json b/package.json index 6b0a379..7d90083 100644 --- a/package.json +++ b/package.json @@ -99,6 +99,7 @@ "migrate:h5-experience-v1": "node scripts/migrate-h5-experience-v1.mjs", "verify:experience-v1": "node --test experience-schema.test.mjs experience-service.test.mjs experience-extractor.test.mjs", "verify:experience-agent-run-local": "node scripts/verify-experience-agent-run-local.mjs", + "verify:experience-reflect-local": "node scripts/verify-experience-reflect-local.mjs", "test:scenario:john4-diet": "node scripts/run-scenario-test.mjs --scenario john4-children-hobby-diet-update", "test": "node --test api-core-retry.test.mjs auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-voice-reco.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs scheduled-task-intent.test.mjs scheduled-task-service.test.mjs scheduled-task-executor.test.mjs scheduled-task-worker.test.mjs scheduled-task-worker-config.test.mjs wechat/handlers/scheduled-task.test.mjs capabilities.test.mjs policies.test.mjs server/portal-api-auth-middleware.test.mjs server/portal-config-routes.test.mjs server/portal-plaza-discovery-routes.test.mjs server/portal-runtime-routes.test.mjs server/portal-gateway-services-bootstrap.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs services/orchestrator/admin-config.test.mjs services/orchestrator/contracts.test.mjs services/orchestrator/checkpoint.test.mjs services/orchestrator/runtime.test.mjs services/orchestrator/app.test.mjs services/orchestrator/server.test.mjs services/orchestrator/shadow-dispatcher.test.mjs services/orchestrator/shadow-observer.test.mjs services/orchestrator/observability.test.mjs services/orchestrator/executor-gateway.test.mjs services/orchestrator/executor-job-store.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-long-image.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-save-service.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-wechat-html-delivery.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-artifact-service.test.mjs mindspace-conversation-package-audit.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-workspace-publication-delivery-service.test.mjs mindspace-workspace-tool-service.test.mjs mindspace-mcp-scoped-token.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", diff --git a/scripts/verify-experience-reflect-local.mjs b/scripts/verify-experience-reflect-local.mjs new file mode 100644 index 0000000..f59d117 --- /dev/null +++ b/scripts/verify-experience-reflect-local.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +/** + * Local acceptance: seed task_outcome rows, run reflect(), verify execution_pattern. + * + * Usage: + * node scripts/verify-experience-reflect-local.mjs + * node scripts/verify-experience-reflect-local.mjs --seed-only + * node scripts/verify-experience-reflect-local.mjs --reflect-only + */ +import { loadH5Environment } from './load-env.mjs'; +import { createDbPool } from '../db.mjs'; +import { createExperienceService } from '../experience-service.mjs'; + +const PROBLEM = 'Experience reflect local acceptance probe'; + +function parseArgs(argv) { + return { + seedOnly: argv.includes('--seed-only'), + reflectOnly: argv.includes('--reflect-only'), + }; +} + +async function countRows(pool, { kind = null, status = 'active' } = {}) { + const clauses = ['status = ?']; + const params = [status]; + if (kind) { + clauses.push('kind = ?'); + params.push(kind); + } + const [rows] = await pool.query( + `SELECT COUNT(*) AS count FROM h5_experience WHERE ${clauses.join(' AND ')}`, + params, + ); + return Number(rows[0]?.count ?? 0); +} + +async function seedTaskOutcomes(service, count = 3) { + const created = []; + for (let i = 0; i < count; i += 1) { + created.push(await service.record({ + kind: 'task_outcome', + title: `Reflect probe ${i + 1}`, + body: `Local acceptance body ${i + 1}`, + problem: PROBLEM, + result: 'success', + confidence: 0.75 + i * 0.05, + environment: { runtime: 'local-dev', components: ['portal', 'goose'] }, + action: { executor: 'goose', steps: [`step-${i + 1}`] }, + tags: ['reflect-acceptance'], + })); + } + return created; +} + +async function main() { + loadH5Environment(import.meta.dirname); + const options = parseArgs(process.argv.slice(2)); + const pool = createDbPool(); + const service = createExperienceService(pool, { productEventsPool: pool }); + + try { + const beforePatterns = await countRows(pool, { kind: 'execution_pattern' }); + const beforeActiveOutcomes = await countRows(pool, { kind: 'task_outcome', status: 'active' }); + + if (!options.reflectOnly) { + const seeded = await seedTaskOutcomes(service, 3); + console.log(`[reflect-acceptance] seeded ${seeded.length} task_outcome rows`); + } + + if (!options.seedOnly) { + const result = await service.reflect({ minGroupSize: 3 }); + console.log('[reflect-acceptance] reflect result:', JSON.stringify({ + ok: result.ok, + created: result.created, + archived: result.archived, + scope: result.scope, + })); + if (!result.created) { + console.log('[reflect-acceptance] note: no new pattern (group may already exist or below threshold)'); + } + } + + const afterPatterns = await countRows(pool, { kind: 'execution_pattern' }); + const afterArchivedOutcomes = await countRows(pool, { kind: 'task_outcome', status: 'archived' }); + const [patterns] = await pool.query( + `SELECT id, title, kind, result, confidence, created_at + FROM h5_experience + WHERE kind = 'execution_pattern' AND problem = ? + ORDER BY created_at DESC + LIMIT 3`, + [PROBLEM], + ); + + console.log('[reflect-acceptance] summary:', JSON.stringify({ + executionPatternsBefore: beforePatterns, + executionPatternsAfter: afterPatterns, + activeTaskOutcomesBefore: beforeActiveOutcomes, + archivedTaskOutcomesAfter: afterArchivedOutcomes, + latestPatterns: patterns, + }, null, 2)); + + const [events] = await pool.query( + `SELECT event_type, data_json, created_at + FROM h5_memory_v2_product_events + WHERE event_type IN ('experience_saved', 'experience_recalled') + ORDER BY created_at DESC + LIMIT 5`, + ).catch(() => [[]]); + + if (events.length > 0) { + console.log('[reflect-acceptance] recent product events:', events); + } else { + console.log('[reflect-acceptance] product events table empty or unavailable (ok on fresh DB)'); + } + + const ok = afterPatterns >= beforePatterns && patterns.length > 0; + if (!ok && !options.seedOnly) { + process.exitCode = 1; + console.error('[reflect-acceptance] FAILED: expected at least one execution_pattern for probe problem'); + } else { + console.log('[reflect-acceptance] OK'); + } + } finally { + await pool.end(); + } +} + +main().catch((error) => { + console.error('[reflect-acceptance] error:', error instanceof Error ? error.message : error); + process.exit(1); +});