john
e90a2da6cb
Recover WIP modules from stash so admin can persist capability config, observe chat writes in shadow mode, and review persisted candidates. Co-authored-by: Cursor <cursoragent@cursor.com>
233 lines
9.3 KiB
JavaScript
233 lines
9.3 KiB
JavaScript
import crypto from 'node:crypto';
|
||
import { deriveUserFacingText } from './conversation-display.mjs';
|
||
|
||
const TRUE_VALUES = new Set(['1', 'true', 'yes', 'on']);
|
||
const FALSE_VALUES = new Set(['0', 'false', 'no', 'off']);
|
||
const SECRET_PATTERNS = [
|
||
/\b(?:api[_-]?key|access[_-]?token|secret|password|passwd)\b\s*[:=]/i,
|
||
/\bsk-[a-z0-9_-]{12,}\b/i,
|
||
/-----BEGIN [A-Z ]+PRIVATE KEY-----/,
|
||
/(?:密码|口令|密钥|令牌)\s*[：:=]\s*\S+/i,
|
||
];
|
||
const TRIVIAL_PATTERNS = [
|
||
/^(?:你好|您好|谢谢|好的|可以|收到|再见|hi|hello|thanks)[！!。.\s]*$/i,
|
||
/^(?:查一下|搜一下|看看|继续|下一步)[。.!！?？\s]*$/i,
|
||
];
|
||
|
||
function readFlag(env, key, fallback = false) {
|
||
const raw = env?.[key];
|
||
if (raw == null || raw === '') return fallback;
|
||
const normalized = String(raw).trim().toLowerCase();
|
||
if (TRUE_VALUES.has(normalized)) return true;
|
||
if (FALSE_VALUES.has(normalized)) return false;
|
||
return fallback;
|
||
}
|
||
|
||
function readNumber(env, key, fallback, { min = -Infinity, max = Infinity } = {}) {
|
||
const value = Number(env?.[key]);
|
||
if (!Number.isFinite(value)) return fallback;
|
||
return Math.min(max, Math.max(min, value));
|
||
}
|
||
|
||
function normalizeText(value, maxLength = 2000) {
|
||
const text = String(value ?? '').replace(/\s+/g, ' ').trim();
|
||
if (!text) return '';
|
||
return text.length > maxLength ? text.slice(0, maxLength) : text;
|
||
}
|
||
|
||
function messageText(message) {
|
||
if (typeof message === 'string') return normalizeText(message);
|
||
if (!message || typeof message !== 'object') return '';
|
||
const displayText = message.displayText
|
||
?? message.display_text
|
||
?? message.metadata?.displayText
|
||
?? message.metadata?.display_text;
|
||
if (typeof displayText === 'string' && displayText.trim()) {
|
||
return normalizeText(displayText);
|
||
}
|
||
const direct = message.text ?? message.content ?? message.message ?? '';
|
||
if (typeof direct === 'string') return normalizeText(deriveUserFacingText(direct));
|
||
if (Array.isArray(direct)) {
|
||
return normalizeText(deriveUserFacingText(
|
||
direct.map((item) => item?.text ?? item?.content ?? '').join('\n'),
|
||
));
|
||
}
|
||
return '';
|
||
}
|
||
|
||
function stableHash(value) {
|
||
return crypto.createHash('sha256').update(String(value)).digest('hex');
|
||
}
|
||
|
||
function classify(text) {
|
||
const rules = [
|
||
{ type: 'episodic', importance: 0.95, confidence: 0.98, pattern: /(?:请记住|记住|以后要|从现在起)/i, reason: 'explicit_memory_request' },
|
||
{ type: 'episodic', importance: 0.9, confidence: 0.9, pattern: /(?:决定|确定|统一采用|必须|不允许|禁止|最终选择)/i, reason: 'decision_signal' },
|
||
{ type: 'preference', importance: 0.8, confidence: 0.88, pattern: /(?:我喜欢|我偏好|我不喜欢|我的习惯|倾向于|更希望)/i, reason: 'preference_signal' },
|
||
{ type: 'goal', importance: 0.85, confidence: 0.86, pattern: /(?:长期目标|当前目标|目标是|计划要|正在建设|准备实现|希望长期)/i, reason: 'goal_signal' },
|
||
{ type: 'semantic', importance: 0.75, confidence: 0.82, pattern: /(?:技术栈|架构原则|项目使用|系统采用|仓库位于|运行在)/i, reason: 'stable_fact_signal' },
|
||
];
|
||
return rules.find((rule) => rule.pattern.test(text)) ?? null;
|
||
}
|
||
|
||
export function resolvePersonalShadowConfig(env = process.env) {
|
||
const candidateEnabled = readFlag(env, 'MEMORY_CANDIDATE_ENABLED', false);
|
||
const requestedMode = normalizeText(env?.MEMORY_CANDIDATE_MODE || (candidateEnabled ? 'shadow' : 'off'), 20).toLowerCase();
|
||
return {
|
||
enabled: candidateEnabled && requestedMode !== 'off',
|
||
requestedMode,
|
||
effectiveMode: candidateEnabled && requestedMode !== 'off' ? 'shadow' : 'off',
|
||
minImportance: readNumber(env, 'MEMORY_CANDIDATE_MIN_IMPORTANCE', 0.7, { min: 0, max: 1 }),
|
||
minConfidence: readNumber(env, 'MEMORY_CANDIDATE_MIN_CONFIDENCE', 0.8, { min: 0, max: 1 }),
|
||
maxPending: Math.round(readNumber(env, 'MEMORY_CANDIDATE_MAX_PENDING', 500, { min: 1, max: 5000 })),
|
||
policyEnabled: readFlag(env, 'MEMORY_POLICY_ENABLED', true),
|
||
saveExplicit: readFlag(env, 'MEMORY_POLICY_SAVE_EXPLICIT', true),
|
||
rejectSensitive: readFlag(env, 'MEMORY_POLICY_REJECT_SENSITIVE', true),
|
||
requireEvidence: readFlag(env, 'MEMORY_POLICY_REQUIRE_EVIDENCE', true),
|
||
healthEnabled: readFlag(env, 'MEMORY_PLUGIN_HEALTH_ENABLED', true),
|
||
};
|
||
}
|
||
|
||
export function createPersonalMemoryShadowPipeline({ env = process.env, now = () => Date.now(), store = null } = {}) {
|
||
const config = resolvePersonalShadowConfig(env);
|
||
const candidates = [];
|
||
const candidateHashes = new Set();
|
||
const metrics = {
|
||
runs: 0,
|
||
observedMessages: 0,
|
||
accepted: 0,
|
||
rejected: 0,
|
||
deduped: 0,
|
||
errors: 0,
|
||
lastRunAt: null,
|
||
lastError: null,
|
||
};
|
||
|
||
function reject(reason) {
|
||
metrics.rejected += 1;
|
||
return { accepted: false, reason };
|
||
}
|
||
|
||
function evaluate({ userId, sessionId, message, index }) {
|
||
const text = messageText(message);
|
||
if (!text || text.length < 8) return reject('too_short');
|
||
if (TRIVIAL_PATTERNS.some((pattern) => pattern.test(text))) return reject('trivial');
|
||
if (config.rejectSensitive && SECRET_PATTERNS.some((pattern) => pattern.test(text))) {
|
||
return reject('sensitive_content');
|
||
}
|
||
const classification = classify(text);
|
||
if (!classification) return reject('no_durable_signal');
|
||
if (classification.reason === 'explicit_memory_request' && !config.saveExplicit) {
|
||
return reject('explicit_memory_disabled');
|
||
}
|
||
if (config.policyEnabled && classification.importance < config.minImportance) {
|
||
return reject('below_importance_threshold');
|
||
}
|
||
if (config.policyEnabled && classification.confidence < config.minConfidence) {
|
||
return reject('below_confidence_threshold');
|
||
}
|
||
const contentHash = stableHash(`${userId}\n${classification.type}\n${text.toLowerCase()}`);
|
||
if (candidateHashes.has(contentHash)) {
|
||
metrics.deduped += 1;
|
||
return { accepted: false, reason: 'duplicate' };
|
||
}
|
||
const capturedAt = now();
|
||
const evidence = {
|
||
sourceType: 'message',
|
||
sourceId: `${sessionId || 'unknown'}:${index}`,
|
||
evidenceHash: stableHash(`${sessionId || ''}\n${index}\n${text}`),
|
||
capturedAt,
|
||
};
|
||
if (config.requireEvidence && !evidence.sourceId) return reject('evidence_required');
|
||
const candidate = {
|
||
id: `pmc_${contentHash.slice(0, 24)}`,
|
||
userId: String(userId),
|
||
sessionId: sessionId ? String(sessionId) : null,
|
||
memoryType: classification.type,
|
||
content: text,
|
||
importance: classification.importance,
|
||
confidence: classification.confidence,
|
||
status: 'candidate',
|
||
policyReason: classification.reason,
|
||
evidence,
|
||
createdAt: capturedAt,
|
||
};
|
||
candidateHashes.add(contentHash);
|
||
candidates.push(candidate);
|
||
while (candidates.length > config.maxPending) {
|
||
const removed = candidates.shift();
|
||
if (removed) candidateHashes.delete(stableHash(`${removed.userId}\n${removed.memoryType}\n${removed.content.toLowerCase()}`));
|
||
}
|
||
metrics.accepted += 1;
|
||
return { accepted: true, candidate };
|
||
}
|
||
|
||
async function observeWrite({ userId, sessionId, messages = [] } = {}) {
|
||
if (!config.enabled) return { enabled: false, skipped: true, reason: 'disabled' };
|
||
metrics.runs += 1;
|
||
metrics.lastRunAt = now();
|
||
if (!userId || !Array.isArray(messages)) {
|
||
return { enabled: true, skipped: true, reason: 'invalid_input' };
|
||
}
|
||
try {
|
||
const results = [];
|
||
for (let index = 0; index < messages.length; index += 1) {
|
||
const message = messages[index];
|
||
const role = normalizeText(message?.role ?? message?.sender ?? '', 30).toLowerCase();
|
||
if (role && !['user', 'human'].includes(role)) continue;
|
||
metrics.observedMessages += 1;
|
||
const result = evaluate({ userId, sessionId, message, index });
|
||
if (result.accepted && store?.saveCandidate) {
|
||
const persisted = await store.saveCandidate(result.candidate);
|
||
if (persisted?.inserted === false) {
|
||
metrics.deduped += 1;
|
||
}
|
||
}
|
||
results.push(result);
|
||
}
|
||
return {
|
||
enabled: true,
|
||
skipped: false,
|
||
mode: config.effectiveMode,
|
||
observed: results.length,
|
||
accepted: results.filter((result) => result.accepted).length,
|
||
rejected: results.filter((result) => !result.accepted).length,
|
||
};
|
||
} catch (err) {
|
||
metrics.errors += 1;
|
||
metrics.lastError = err instanceof Error ? err.message : String(err);
|
||
return { enabled: true, skipped: true, reason: 'pipeline_failure' };
|
||
}
|
||
}
|
||
|
||
function getStatus() {
|
||
return {
|
||
enabled: config.enabled,
|
||
requestedMode: config.requestedMode,
|
||
effectiveMode: config.effectiveMode,
|
||
phase: 'shadow-candidate-v1',
|
||
injectionEnabled: false,
|
||
persistence: store?.saveCandidate ? 'mysql' : 'bounded-memory',
|
||
pendingCandidates: candidates.length,
|
||
health: {
|
||
enabled: config.healthEnabled,
|
||
state: metrics.errors > 0 ? 'degraded' : 'healthy',
|
||
...metrics,
|
||
},
|
||
policy: {
|
||
enabled: config.policyEnabled,
|
||
minImportance: config.minImportance,
|
||
minConfidence: config.minConfidence,
|
||
rejectSensitive: config.rejectSensitive,
|
||
requireEvidence: config.requireEvidence,
|
||
},
|
||
};
|
||
}
|
||
|
||
function listCandidates({ limit = 50 } = {}) {
|
||
return candidates.slice(-Math.max(1, Math.min(200, Number(limit) || 50))).reverse();
|
||
}
|
||
|
||
return { config, observeWrite, getStatus, listCandidates };
|
||
}
|