import crypto from 'node:crypto'; import { deriveUserFacingText } from './conversation-display.mjs'; const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']); const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']); const SECRET_PATTERNS = [ /\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i, /\bsk-[a-z0-9_-]{12,}\b/i, /-----BEGIN [A-Z ]+PRIVATE KEY-----/, /(?:密码|口令|密钥|令牌)\s*[::=]\s*\S+/i, ]; const TRIVIAL_PATTERNS = [ /^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[!!。.\s]*$/i, /^(?:查一下|搜一下|看看|继续|下一步)[。.!!??\s]*$/i, ]; function readFlag(env, key, fallback = false) { const raw = env?.[key]; if (raw == null || raw === '') return fallback; const normalized = String(raw).trim().toLowerCase(); if (TRUE_VALUES.has(normalized)) return true; if (FALSE_VALUES.has(normalized)) return false; return fallback; } function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) { const value = Number(env?.[key]); if (!Number.isFinite(value)) return fallback; return Math.min(max, Math.max(min, value)); } function normalizeText(value, maxLength = 2000) { const text = String(value ?? '').replace(/\s+/g, ' ').trim(); if (!text) return ''; return text.length > maxLength ? text.slice(0, maxLength) : text; } function messageText(message) { if (typeof message === 'string') return normalizeText(message); if (!message || typeof message !== 'object') return ''; const displayText = message.displayText ?? message.display_text ?? message.metadata?.displayText ?? message.metadata?.display_text; if (typeof displayText === 'string' && displayText.trim()) { return normalizeText(displayText); } const direct = message.text ?? message.content ?? message.message ?? ''; if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct)); if (Array.isArray(direct)) { return normalizeText(deriveUserFacingText( direct.map((item) => item?.text ?? item?.content ?? '').join('\n'), )); } return ''; } function stableHash(value) { return crypto.createHash('sha256').update(String(value)).digest('hex'); } function classify(text) { const rules = [ { type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' }, { type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' }, { type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' }, { type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' }, { type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' }, ]; return rules.find((rule) => rule.pattern.test(text)) ?? null; } const CANDIDATE_MODES = new Set(['off', 'shadow', 'canary', 'active']); function normalizeCandidateMode(value, fallback = 'shadow') { const normalized = normalizeText(value, 20).toLowerCase(); return CANDIDATE_MODES.has(normalized) ? normalized : fallback; } export function shouldAutoAcceptCandidate(candidate, config) { if (!candidate || !config?.enabled) return false; const mode = normalizeCandidateMode(config.requestedMode, 'shadow'); if (mode === 'shadow' || mode === 'off') return false; if (mode === 'active') return true; if (candidate.policyReason === 'explicit_memory_request') return true; return Number(candidate.confidence) >= Math.max(Number(config.minConfidence) || 0, 0.9); } export function resolvePersonalShadowConfig(env = process.env) { const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false); const requestedMode = normalizeCandidateMode( env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'active' : 'off'), candidateEnabled ? 'active' : 'off', ); const enabled = candidateEnabled && requestedMode !== 'off'; return { enabled, requestedMode, effectiveMode: enabled ? requestedMode : 'off', autoReviewEnabled: enabled && requestedMode !== 'shadow', minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }), minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }), maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })), policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true), saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true), rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true), requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true), healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true), }; } export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) { const config = resolvePersonalShadowConfig(env); const candidates = []; const candidateHashes = new Set(); const metrics = { runs: 0, observedMessages: 0, accepted: 0, autoReviewed: 0, pendingReview: 0, rejected: 0, deduped: 0, errors: 0, lastRunAt: null, lastError: null, }; function reject(reason) { metrics.rejected += 1; return { accepted: false, reason }; } function evaluate({ userId, sessionId, message, index }) { const text = messageText(message); if (!text || text.length < 8) return reject('too_short'); if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial'); if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) { return reject('sensitive_content'); } const classification = classify(text); if (!classification) return reject('no_durable_signal'); if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) { return reject('explicit_memory_disabled'); } if (config.policyEnabled && classification.importance < config.minImportance) { return reject('below_importance_threshold'); } if (config.policyEnabled && classification.confidence < config.minConfidence) { return reject('below_confidence_threshold'); } const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`); if (candidateHashes.has(contentHash)) { metrics.deduped += 1; return { accepted: false, reason: 'duplicate' }; } const capturedAt = now(); const evidence = { sourceType: 'message', sourceId: `${sessionId || 'unknown'}:${index}`, evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`), capturedAt, }; if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required'); const candidate = { id: `pmc_${contentHash.slice(0, 24)}`, userId: String(userId), sessionId: sessionId ? String(sessionId) : null, memoryType: classification.type, content: text, importance: classification.importance, confidence: classification.confidence, status: 'candidate', policyReason: classification.reason, evidence, createdAt: capturedAt, }; candidateHashes.add(contentHash); candidates.push(candidate); while (candidates.length > config.maxPending) { const removed = candidates.shift(); if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`)); } metrics.accepted += 1; return { accepted: true, candidate }; } async function observeWrite({ userId, sessionId, messages = [] } = {}) { if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' }; metrics.runs += 1; metrics.lastRunAt = now(); if (!userId || !Array.isArray(messages)) { return { enabled: true, skipped: true, reason: 'invalid_input' }; } try { const results = []; for (let index = 0; index < messages.length; index += 1) { const message = messages[index]; const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase(); if (role && !['user', 'human'].includes(role)) continue; metrics.observedMessages += 1; const result = evaluate({ userId, sessionId, message, index }); if (result.accepted && store?.saveCandidate) { const autoAccept = shouldAutoAcceptCandidate(result.candidate, config); const persisted = await store.saveCandidate(result.candidate, { autoAccept }); if (persisted?.inserted === false) { metrics.deduped += 1; } else if (autoAccept) { result.candidate.status = 'accepted'; metrics.autoReviewed += 1; } else { metrics.pendingReview += 1; } } else if (result.accepted) { const autoAccept = shouldAutoAcceptCandidate(result.candidate, config); result.candidate.status = autoAccept ? 'accepted' : 'candidate'; if (autoAccept) metrics.autoReviewed += 1; else metrics.pendingReview += 1; } results.push(result); } return { enabled: true, skipped: false, mode: config.effectiveMode, autoReviewEnabled: config.autoReviewEnabled, observed: results.length, accepted: results.filter((result) => result.accepted).length, autoReviewed: results.filter((result) => result.accepted && result.candidate?.status === 'accepted').length, pendingReview: results.filter((result) => result.accepted && result.candidate?.status === 'candidate').length, rejected: results.filter((result) => !result.accepted).length, }; } catch (err) { metrics.errors += 1; metrics.lastError = err instanceof Error ? err.message : String(err); return { enabled: true, skipped: true, reason: 'pipeline_failure' }; } } function getStatus() { return { enabled: config.enabled, requestedMode: config.requestedMode, effectiveMode: config.effectiveMode, phase: config.autoReviewEnabled ? 'auto-review-v1' : 'shadow-candidate-v1', autoReviewEnabled: config.autoReviewEnabled, injectionEnabled: false, persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory', pendingCandidates: candidates.length, health: { enabled: config.healthEnabled, state: metrics.errors > 0 ? 'degraded' : 'healthy', ...metrics, }, policy: { enabled: config.policyEnabled, minImportance: config.minImportance, minConfidence: config.minConfidence, rejectSensitive: config.rejectSensitive, requireEvidence: config.requireEvidence, }, }; } function listCandidates({ limit = 50 } = {}) { return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse(); } return { config, observeWrite, getStatus, listCandidates }; }