feat(memory-v2): restore personal shadow pipeline and candidate store

Recover WIP modules from stash so admin can persist capability config, observe chat writes in shadow mode, and review persisted candidates. Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-13 13:59:51 +08:00
parent 249b0697cb
commit e90a2da6cb
9 changed files with 773 additions and 1 deletions
@@ -0,0 +1,232 @@
+import crypto from 'node:crypto';
+import { deriveUserFacingText } from './conversation-display.mjs';
+
+const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']);
+const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
+const SECRET_PATTERNS = [
+  /\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i,
+  /\bsk-[a-z0-9_-]{12,}\b/i,
+  /-----BEGIN [A-Z ]+PRIVATE KEY-----/,
+  /(?:密码|口令|密钥|令牌)\s*[：:=]\s*\S+/i,
+];
+const TRIVIAL_PATTERNS = [
+  /^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[！!。.\s]*$/i,
+  /^(?:查一下|搜一下|看看|继续|下一步)[。.!！?？\s]*$/i,
+];
+
+function readFlag(env, key, fallback = false) {
+  const raw = env?.[key];
+  if (raw == null || raw === '') return fallback;
+  const normalized = String(raw).trim().toLowerCase();
+  if (TRUE_VALUES.has(normalized)) return true;
+  if (FALSE_VALUES.has(normalized)) return false;
+  return fallback;
+}
+
+function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) {
+  const value = Number(env?.[key]);
+  if (!Number.isFinite(value)) return fallback;
+  return Math.min(max, Math.max(min, value));
+}
+
+function normalizeText(value, maxLength = 2000) {
+  const text = String(value ?? '').replace(/\s+/g, ' ').trim();
+  if (!text) return '';
+  return text.length > maxLength ? text.slice(0, maxLength) : text;
+}
+
+function messageText(message) {
+  if (typeof message === 'string') return normalizeText(message);
+  if (!message || typeof message !== 'object') return '';
+  const displayText = message.displayText
+    ?? message.display_text
+    ?? message.metadata?.displayText
+    ?? message.metadata?.display_text;
+  if (typeof displayText === 'string' && displayText.trim()) {
+    return normalizeText(displayText);
+  }
+  const direct = message.text ?? message.content ?? message.message ?? '';
+  if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct));
+  if (Array.isArray(direct)) {
+    return normalizeText(deriveUserFacingText(
+      direct.map((item) => item?.text ?? item?.content ?? '').join('\n'),
+    ));
+  }
+  return '';
+}
+
+function stableHash(value) {
+  return crypto.createHash('sha256').update(String(value)).digest('hex');
+}
+
+function classify(text) {
+  const rules = [
+    { type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' },
+    { type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' },
+    { type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' },
+    { type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' },
+    { type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' },
+  ];
+  return rules.find((rule) => rule.pattern.test(text)) ?? null;
+}
+
+export function resolvePersonalShadowConfig(env = process.env) {
+  const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false);
+  const requestedMode = normalizeText(env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'shadow' : 'off'), 20).toLowerCase();
+  return {
+    enabled: candidateEnabled && requestedMode !== 'off',
+    requestedMode,
+    effectiveMode: candidateEnabled && requestedMode !== 'off' ? 'shadow' : 'off',
+    minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }),
+    minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }),
+    maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })),
+    policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true),
+    saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true),
+    rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true),
+    requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true),
+    healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true),
+  };
+}
+
+export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) {
+  const config = resolvePersonalShadowConfig(env);
+  const candidates = [];
+  const candidateHashes = new Set();
+  const metrics = {
+    runs: 0,
+    observedMessages: 0,
+    accepted: 0,
+    rejected: 0,
+    deduped: 0,
+    errors: 0,
+    lastRunAt: null,
+    lastError: null,
+  };
+
+  function reject(reason) {
+    metrics.rejected += 1;
+    return { accepted: false, reason };
+  }
+
+  function evaluate({ userId, sessionId, message, index }) {
+    const text = messageText(message);
+    if (!text || text.length < 8) return reject('too_short');
+    if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial');
+    if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) {
+      return reject('sensitive_content');
+    }
+    const classification = classify(text);
+    if (!classification) return reject('no_durable_signal');
+    if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) {
+      return reject('explicit_memory_disabled');
+    }
+    if (config.policyEnabled && classification.importance < config.minImportance) {
+      return reject('below_importance_threshold');
+    }
+    if (config.policyEnabled && classification.confidence < config.minConfidence) {
+      return reject('below_confidence_threshold');
+    }
+    const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`);
+    if (candidateHashes.has(contentHash)) {
+      metrics.deduped += 1;
+      return { accepted: false, reason: 'duplicate' };
+    }
+    const capturedAt = now();
+    const evidence = {
+      sourceType: 'message',
+      sourceId: `${sessionId || 'unknown'}:${index}`,
+      evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`),
+      capturedAt,
+    };
+    if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required');
+    const candidate = {
+      id: `pmc_${contentHash.slice(0, 24)}`,
+      userId: String(userId),
+      sessionId: sessionId ? String(sessionId) : null,
+      memoryType: classification.type,
+      content: text,
+      importance: classification.importance,
+      confidence: classification.confidence,
+      status: 'candidate',
+      policyReason: classification.reason,
+      evidence,
+      createdAt: capturedAt,
+    };
+    candidateHashes.add(contentHash);
+    candidates.push(candidate);
+    while (candidates.length > config.maxPending) {
+      const removed = candidates.shift();
+      if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`));
+    }
+    metrics.accepted += 1;
+    return { accepted: true, candidate };
+  }
+
+  async function observeWrite({ userId, sessionId, messages = [] } = {}) {
+    if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' };
+    metrics.runs += 1;
+    metrics.lastRunAt = now();
+    if (!userId || !Array.isArray(messages)) {
+      return { enabled: true, skipped: true, reason: 'invalid_input' };
+    }
+    try {
+      const results = [];
+      for (let index = 0; index < messages.length; index += 1) {
+        const message = messages[index];
+        const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase();
+        if (role && !['user', 'human'].includes(role)) continue;
+        metrics.observedMessages += 1;
+        const result = evaluate({ userId, sessionId, message, index });
+        if (result.accepted && store?.saveCandidate) {
+          const persisted = await store.saveCandidate(result.candidate);
+          if (persisted?.inserted === false) {
+            metrics.deduped += 1;
+          }
+        }
+        results.push(result);
+      }
+      return {
+        enabled: true,
+        skipped: false,
+        mode: config.effectiveMode,
+        observed: results.length,
+        accepted: results.filter((result) => result.accepted).length,
+        rejected: results.filter((result) => !result.accepted).length,
+      };
+    } catch (err) {
+      metrics.errors += 1;
+      metrics.lastError = err instanceof Error ? err.message : String(err);
+      return { enabled: true, skipped: true, reason: 'pipeline_failure' };
+    }
+  }
+
+  function getStatus() {
+    return {
+      enabled: config.enabled,
+      requestedMode: config.requestedMode,
+      effectiveMode: config.effectiveMode,
+      phase: 'shadow-candidate-v1',
+      injectionEnabled: false,
+      persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory',
+      pendingCandidates: candidates.length,
+      health: {
+        enabled: config.healthEnabled,
+        state: metrics.errors > 0 ? 'degraded' : 'healthy',
+        ...metrics,
+      },
+      policy: {
+        enabled: config.policyEnabled,
+        minImportance: config.minImportance,
+        minConfidence: config.minConfidence,
+        rejectSensitive: config.rejectSensitive,
+        requireEvidence: config.requireEvidence,
+      },
+    };
+  }
+
+  function listCandidates({ limit = 50 } = {}) {
+    return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse();
+  }
+
+  return { config, observeWrite, getStatus, listCandidates };
+}