Add User Model Service and Temporal Recall for MeMind V0.1.
Introduce UMS ingest/snapshot pipeline, Context Planner with multi-source recall, runtime context injection, canonical user mapping, and session snapshot loading on auth/me. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
function newId() {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
|
||||
function nowMs() {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function slugEntityName(name) {
|
||||
return String(name).trim().slice(0, 128);
|
||||
}
|
||||
|
||||
/**
|
||||
* V0.1:从 term_frequency signals 推断 project/focus candidates
|
||||
* @param {import('mysql2/promise').Pool} pool
|
||||
* @param {string} userId
|
||||
*/
|
||||
export async function mergeCandidatesFromSignals(pool, userId) {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT signal_id, dimension_key, value_json, evidence_ids, window_end
|
||||
FROM um_signals
|
||||
WHERE user_id = ? AND signal_type = 'term_frequency'
|
||||
ORDER BY window_end DESC
|
||||
LIMIT 500`,
|
||||
[userId],
|
||||
);
|
||||
|
||||
const termCounts = new Map();
|
||||
for (const row of rows) {
|
||||
const term = row.dimension_key.replace(/^term:/, '');
|
||||
const value = typeof row.value_json === 'string' ? JSON.parse(row.value_json) : row.value_json;
|
||||
const evidenceIds =
|
||||
typeof row.evidence_ids === 'string' ? JSON.parse(row.evidence_ids) : row.evidence_ids;
|
||||
const prev = termCounts.get(term) ?? { count: 0, signal_ids: [], evidence_ids: [] };
|
||||
prev.count += Number(value.count ?? 0);
|
||||
prev.signal_ids.push(row.signal_id);
|
||||
prev.evidence_ids.push(...(evidenceIds ?? []));
|
||||
termCounts.set(term, prev);
|
||||
}
|
||||
|
||||
let touched = 0;
|
||||
const ts = nowMs();
|
||||
for (const [term, stats] of termCounts) {
|
||||
if (stats.count < 5) continue;
|
||||
if (term.length < 2) continue;
|
||||
|
||||
const isProjectLike = /^[A-Z][a-zA-Z0-9]+$/.test(term) || term.includes('Input') || term.includes('Mind');
|
||||
const candidateType = isProjectLike ? 'project' : 'focus';
|
||||
const confidence = Math.min(0.99, 0.4 + stats.count * 0.03);
|
||||
const promotionScore = Math.min(0.99, confidence * Math.min(1, stats.count / 20));
|
||||
const hypothesis = isProjectLike
|
||||
? { type: 'project', name: term, status: 'active' }
|
||||
: { type: 'focus', topic: term };
|
||||
|
||||
const [existing] = await pool.query(
|
||||
isProjectLike
|
||||
? `SELECT candidate_id FROM um_candidates
|
||||
WHERE user_id = ? AND candidate_type = 'project'
|
||||
AND JSON_UNQUOTE(JSON_EXTRACT(hypothesis_json, '$.name')) = ?
|
||||
LIMIT 1`
|
||||
: `SELECT candidate_id FROM um_candidates
|
||||
WHERE user_id = ? AND candidate_type = 'focus'
|
||||
AND JSON_UNQUOTE(JSON_EXTRACT(hypothesis_json, '$.topic')) = ?
|
||||
LIMIT 1`,
|
||||
[userId, term],
|
||||
);
|
||||
|
||||
const status = promotionScore >= 0.8 ? 'accepted' : promotionScore >= 0.55 ? 'open' : 'observed';
|
||||
const uniqueEvidence = [...new Set(stats.evidence_ids)].slice(0, 200);
|
||||
|
||||
if (existing[0]) {
|
||||
await pool.query(
|
||||
`UPDATE um_candidates
|
||||
SET promotion_score = ?, confidence = ?, status = ?, signal_ids = ?, evidence_ids = ?,
|
||||
last_seen_at = NOW(3), updated_at = ?, version = version + 1
|
||||
WHERE candidate_id = ?`,
|
||||
[
|
||||
promotionScore,
|
||||
confidence,
|
||||
status,
|
||||
JSON.stringify(stats.signal_ids),
|
||||
JSON.stringify(uniqueEvidence),
|
||||
ts,
|
||||
existing[0].candidate_id,
|
||||
],
|
||||
);
|
||||
} else {
|
||||
await pool.query(
|
||||
`INSERT INTO um_candidates
|
||||
(candidate_id, user_id, candidate_type, hypothesis_json, status, promotion_score, confidence,
|
||||
signal_ids, evidence_ids, first_seen_at, last_seen_at, version, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NOW(3), NOW(3), 1, ?, ?)`,
|
||||
[
|
||||
newId(),
|
||||
userId,
|
||||
candidateType,
|
||||
JSON.stringify(hypothesis),
|
||||
status,
|
||||
promotionScore,
|
||||
confidence,
|
||||
JSON.stringify(stats.signal_ids),
|
||||
JSON.stringify(uniqueEvidence),
|
||||
ts,
|
||||
ts,
|
||||
],
|
||||
);
|
||||
}
|
||||
touched += 1;
|
||||
|
||||
if (status === 'accepted' && isProjectLike) {
|
||||
await upsertProjectGraph(pool, userId, term, confidence, uniqueEvidence, promotionScore);
|
||||
}
|
||||
}
|
||||
return touched;
|
||||
}
|
||||
|
||||
async function upsertProjectGraph(pool, userId, name, confidence, evidenceIds, weight) {
|
||||
const ts = nowMs();
|
||||
const canonical = slugEntityName(name);
|
||||
const [entities] = await pool.query(
|
||||
`SELECT entity_id FROM um_entities WHERE user_id = ? AND entity_type = 'project' AND canonical_name = ? LIMIT 1`,
|
||||
[userId, canonical],
|
||||
);
|
||||
let entityId = entities[0]?.entity_id;
|
||||
if (!entityId) {
|
||||
entityId = newId();
|
||||
await pool.query(
|
||||
`INSERT INTO um_entities (entity_id, user_id, entity_type, canonical_name, status, created_at, updated_at)
|
||||
VALUES (?, ?, 'project', ?, 'active', ?, ?)`,
|
||||
[entityId, userId, canonical, ts, ts],
|
||||
);
|
||||
}
|
||||
|
||||
const valueJson = { name: canonical, status: 'active' };
|
||||
const contentHash = crypto.createHash('sha256').update(JSON.stringify(valueJson)).digest('hex');
|
||||
const [attrs] = await pool.query(
|
||||
`SELECT attribute_id FROM um_attributes
|
||||
WHERE user_id = ? AND entity_id = ? AND attr_key = 'project.status' AND status = 'active'
|
||||
LIMIT 1`,
|
||||
[userId, entityId],
|
||||
);
|
||||
if (attrs[0]) {
|
||||
await pool.query(
|
||||
`UPDATE um_attributes SET confidence = ?, effective_weight = ?, evidence_ids = ?, last_seen_at = NOW(3)
|
||||
WHERE attribute_id = ?`,
|
||||
[confidence, weight, JSON.stringify(evidenceIds.slice(0, 50)), attrs[0].attribute_id],
|
||||
);
|
||||
} else {
|
||||
await pool.query(
|
||||
`INSERT INTO um_attributes
|
||||
(attribute_id, user_id, entity_id, attr_key, value_json, confidence, decay_halflife_days,
|
||||
effective_weight, evidence_ids, first_seen_at, last_seen_at, status, version, content_hash)
|
||||
VALUES (?, ?, ?, 'project.status', ?, ?, 90, ?, ?, NOW(3), NOW(3), 'active', 1, ?)`,
|
||||
[
|
||||
newId(),
|
||||
userId,
|
||||
entityId,
|
||||
JSON.stringify(valueJson),
|
||||
confidence,
|
||||
weight,
|
||||
JSON.stringify(evidenceIds.slice(0, 50)),
|
||||
contentHash,
|
||||
],
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/**
|
||||
* Resolve login user_id → canonical user_id for UMS / Temporal Recall / MeInput queries.
|
||||
*
|
||||
* Env MEMIND_CANONICAL_USER_MAP:
|
||||
* from=to,from2=to2
|
||||
*
|
||||
* Default: tang19821002 → 唐 (wx_ul610et8)
|
||||
*/
|
||||
const DEFAULT_MAP = new Map([
|
||||
[
|
||||
'd0678bbc-2a50-4e08-8bf0-6b6c9301e2d6',
|
||||
'a70ff537-8908-486e-9b6c-042e07cc25db',
|
||||
],
|
||||
]);
|
||||
|
||||
let cachedMap = null;
|
||||
|
||||
function parseCanonicalUserMap(raw) {
|
||||
const map = new Map(DEFAULT_MAP);
|
||||
const text = String(raw ?? '').trim();
|
||||
if (!text) return map;
|
||||
for (const part of text.split(',')) {
|
||||
const pair = part.trim();
|
||||
if (!pair) continue;
|
||||
const eq = pair.indexOf('=');
|
||||
if (eq <= 0) continue;
|
||||
const from = pair.slice(0, eq).trim();
|
||||
const to = pair.slice(eq + 1).trim();
|
||||
if (from && to) map.set(from, to);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function canonicalMap() {
|
||||
if (!cachedMap) {
|
||||
cachedMap = parseCanonicalUserMap(process.env.MEMIND_CANONICAL_USER_MAP);
|
||||
}
|
||||
return cachedMap;
|
||||
}
|
||||
|
||||
/** @param {string | null | undefined} userId */
|
||||
export function resolveCanonicalUserId(userId) {
|
||||
const id = String(userId ?? '').trim();
|
||||
if (!id) return id;
|
||||
return canonicalMap().get(id) ?? id;
|
||||
}
|
||||
|
||||
export function resetCanonicalUserMapCache() {
|
||||
cachedMap = null;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {
|
||||
resolveCanonicalUserId,
|
||||
resetCanonicalUserMapCache,
|
||||
} from './canonical-user.mjs';
|
||||
|
||||
test('resolveCanonicalUserId maps tang19821002 to wx 唐 by default', () => {
|
||||
resetCanonicalUserMapCache();
|
||||
assert.equal(
|
||||
resolveCanonicalUserId('d0678bbc-2a50-4e08-8bf0-6b6c9301e2d6'),
|
||||
'a70ff537-8908-486e-9b6c-042e07cc25db',
|
||||
);
|
||||
assert.equal(
|
||||
resolveCanonicalUserId('a70ff537-8908-486e-9b6c-042e07cc25db'),
|
||||
'a70ff537-8908-486e-9b6c-042e07cc25db',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveCanonicalUserId respects MEMIND_CANONICAL_USER_MAP env', () => {
|
||||
process.env.MEMIND_CANONICAL_USER_MAP = 'user-a=user-b';
|
||||
resetCanonicalUserMapCache();
|
||||
assert.equal(resolveCanonicalUserId('user-a'), 'user-b');
|
||||
delete process.env.MEMIND_CANONICAL_USER_MAP;
|
||||
resetCanonicalUserMapCache();
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import mysql from 'mysql2/promise';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
export function isUmsDatabaseConfigured() {
|
||||
return Boolean(
|
||||
process.env.UMS_DATABASE_URL ||
|
||||
(process.env.UMS_MYSQL_HOST && process.env.UMS_MYSQL_DATABASE),
|
||||
);
|
||||
}
|
||||
|
||||
function poolOptions() {
|
||||
return {
|
||||
waitForConnections: true,
|
||||
connectionLimit: Math.max(1, Number(process.env.UMS_MYSQL_POOL_SIZE ?? 10)),
|
||||
queueLimit: 0,
|
||||
timezone: 'Z',
|
||||
};
|
||||
}
|
||||
|
||||
export function createUmsPool() {
|
||||
if (!isUmsDatabaseConfigured()) {
|
||||
throw new Error('UMS MySQL 未配置,请设置 UMS_DATABASE_URL 或 UMS_MYSQL_*');
|
||||
}
|
||||
if (process.env.UMS_DATABASE_URL) {
|
||||
return mysql.createPool({ uri: process.env.UMS_DATABASE_URL, ...poolOptions() });
|
||||
}
|
||||
return mysql.createPool({
|
||||
host: process.env.UMS_MYSQL_HOST ?? 'localhost',
|
||||
port: Number(process.env.UMS_MYSQL_PORT ?? 3306),
|
||||
user: process.env.UMS_MYSQL_USER ?? 'boot',
|
||||
password: process.env.UMS_MYSQL_PASSWORD ?? '',
|
||||
database: process.env.UMS_MYSQL_DATABASE ?? 'memind_user_model',
|
||||
...poolOptions(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function runUmsSchema(pool) {
|
||||
const schemaPath = path.join(__dirname, '..', 'schemas', 'memind_user_model-v0.sql');
|
||||
const sql = fs.readFileSync(schemaPath, 'utf8');
|
||||
const statements = sql
|
||||
.split(';')
|
||||
.map((statement) =>
|
||||
statement
|
||||
.split('\n')
|
||||
.filter((line) => !line.trim().startsWith('--'))
|
||||
.join('\n')
|
||||
.trim(),
|
||||
)
|
||||
.filter(Boolean);
|
||||
for (const statement of statements) {
|
||||
await pool.query(statement);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
function newId() {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
|
||||
function nowMs() {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function validateEnvelope(item) {
|
||||
const required = [
|
||||
'evidence_id',
|
||||
'user_id',
|
||||
'source_type',
|
||||
'source_ref',
|
||||
'occurred_at',
|
||||
'evidence_type',
|
||||
'payload',
|
||||
'content_hash',
|
||||
'schema_version',
|
||||
];
|
||||
for (const key of required) {
|
||||
if (item[key] === undefined || item[key] === null || item[key] === '') {
|
||||
return { ok: false, reason: `missing ${key}` };
|
||||
}
|
||||
}
|
||||
if (item.schema_version !== 1) return { ok: false, reason: 'unsupported schema_version' };
|
||||
if (item.privacy_level === 'secure_skip') return { ok: false, reason: 'secure_skip' };
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('mysql2/promise').Pool} pool
|
||||
* @param {{ items: object[], dry_run?: boolean }} input
|
||||
*/
|
||||
export async function ingestEvidenceBatch(pool, { items, dry_run = false }) {
|
||||
const accepted = [];
|
||||
const duplicates = [];
|
||||
const rejected = [];
|
||||
|
||||
for (const item of items ?? []) {
|
||||
const check = validateEnvelope(item);
|
||||
if (!check.ok) {
|
||||
rejected.push({ evidence_id: item.evidence_id ?? null, reason: check.reason });
|
||||
continue;
|
||||
}
|
||||
|
||||
const [existing] = await pool.query(
|
||||
`SELECT evidence_id, content_hash FROM um_evidence WHERE user_id = ? AND content_hash = ? LIMIT 1`,
|
||||
[item.user_id, item.content_hash],
|
||||
);
|
||||
if (existing[0]) {
|
||||
duplicates.push(existing[0].evidence_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
const [byId] = await pool.query(`SELECT content_hash FROM um_evidence WHERE evidence_id = ? LIMIT 1`, [
|
||||
item.evidence_id,
|
||||
]);
|
||||
if (byId[0] && byId[0].content_hash !== item.content_hash) {
|
||||
rejected.push({ evidence_id: item.evidence_id, reason: 'evidence_conflict' });
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dry_run) {
|
||||
accepted.push(item.evidence_id);
|
||||
continue;
|
||||
}
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO um_evidence
|
||||
(evidence_id, user_id, source_type, source_ref, evidence_type, occurred_at, received_at,
|
||||
content_hash, schema_version, privacy_level, payload_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?, NOW(3), ?, ?, ?, ?)`,
|
||||
[
|
||||
item.evidence_id,
|
||||
item.user_id,
|
||||
item.source_type,
|
||||
item.source_ref,
|
||||
item.evidence_type,
|
||||
item.occurred_at.replace('T', ' ').replace('Z', '').slice(0, 23),
|
||||
item.content_hash,
|
||||
item.schema_version,
|
||||
item.privacy_level ?? 'normal',
|
||||
JSON.stringify(item.payload),
|
||||
],
|
||||
);
|
||||
accepted.push(item.evidence_id);
|
||||
}
|
||||
|
||||
return { accepted, duplicates, rejected };
|
||||
}
|
||||
|
||||
export async function updateIngestCursor(pool, userId, sourceType, cursorValue) {
|
||||
await pool.query(
|
||||
`INSERT INTO um_ingest_cursors (user_id, source_type, cursor_value, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE cursor_value = VALUES(cursor_value), updated_at = VALUES(updated_at)`,
|
||||
[userId, sourceType, cursorValue, nowMs()],
|
||||
);
|
||||
}
|
||||
|
||||
export { validateEnvelope };
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
import { createUmsPool, isUmsDatabaseConfigured, runUmsSchema } from './db.mjs';
|
||||
|
||||
async function main() {
|
||||
if (!isUmsDatabaseConfigured()) {
|
||||
console.error('UMS 未配置。请设置 UMS_DATABASE_URL 或 UMS_MYSQL_*');
|
||||
process.exit(1);
|
||||
}
|
||||
const pool = createUmsPool();
|
||||
try {
|
||||
await runUmsSchema(pool);
|
||||
console.log('memind_user_model schema v0 migrated successfully.');
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
import { mergeCandidatesFromSignals } from './candidates.mjs';
|
||||
import { ingestEvidenceBatch, updateIngestCursor } from './ingest.mjs';
|
||||
import { extractSignalsFromEnvelope, upsertSignals } from './signals.mjs';
|
||||
import { materializeProfileAndSnapshot } from './snapshot.mjs';
|
||||
|
||||
/**
|
||||
* @param {import('mysql2/promise').Pool} pool
|
||||
* @param {{ items: object[], source_type?: string, dry_run?: boolean }} input
|
||||
*/
|
||||
export async function processIngestBatch(pool, input) {
|
||||
const ingestResult = await ingestEvidenceBatch(pool, input);
|
||||
if (input.dry_run) {
|
||||
return { ...ingestResult, signals_computed: 0, candidates_touched: 0, snapshot: null };
|
||||
}
|
||||
|
||||
const userIds = new Set();
|
||||
for (const id of ingestResult.accepted) {
|
||||
const [rows] = await pool.query(`SELECT user_id, payload_json, evidence_type, evidence_id, occurred_at, source_type, source_ref, content_hash, schema_version, privacy_level FROM um_evidence WHERE evidence_id = ?`, [id]);
|
||||
const row = rows[0];
|
||||
if (!row) continue;
|
||||
userIds.add(row.user_id);
|
||||
const envelope = {
|
||||
evidence_id: row.evidence_id,
|
||||
user_id: row.user_id,
|
||||
source_type: row.source_type,
|
||||
source_ref: row.source_ref,
|
||||
occurred_at: row.occurred_at instanceof Date ? row.occurred_at.toISOString() : row.occurred_at,
|
||||
evidence_type: row.evidence_type,
|
||||
payload: typeof row.payload_json === 'string' ? JSON.parse(row.payload_json) : row.payload_json,
|
||||
content_hash: row.content_hash,
|
||||
schema_version: row.schema_version,
|
||||
privacy_level: row.privacy_level,
|
||||
};
|
||||
const drafts = extractSignalsFromEnvelope(envelope);
|
||||
await upsertSignals(pool, row.user_id, drafts);
|
||||
}
|
||||
|
||||
let signalsComputed = 0;
|
||||
let candidatesTouched = 0;
|
||||
let snapshot = null;
|
||||
|
||||
for (const userId of userIds) {
|
||||
signalsComputed += 1;
|
||||
candidatesTouched += await mergeCandidatesFromSignals(pool, userId);
|
||||
snapshot = await materializeProfileAndSnapshot(pool, userId, { reason: 'ingest_batch' });
|
||||
}
|
||||
|
||||
const sourceType = input.source_type ?? input.items?.[0]?.source_type ?? 'meinput';
|
||||
const lastItem = input.items?.[input.items.length - 1];
|
||||
if (lastItem?.occurred_at && userIds.size === 1) {
|
||||
await updateIngestCursor(pool, [...userIds][0], sourceType, lastItem.occurred_at);
|
||||
}
|
||||
|
||||
return {
|
||||
...ingestResult,
|
||||
signals_computed: signalsComputed,
|
||||
candidates_touched: candidatesTouched,
|
||||
snapshot: snapshot
|
||||
? {
|
||||
profile_version_bumped: true,
|
||||
fast_revision_bumped: true,
|
||||
...snapshot,
|
||||
}
|
||||
: { profile_version_bumped: false, fast_revision_bumped: false },
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { resolveCanonicalUserId } from './canonical-user.mjs';
|
||||
import { getActiveSnapshot } from './snapshot.mjs';
|
||||
|
||||
/**
|
||||
* Session bootstrap payload for /auth/me (RFC: 5~10KB snapshot at session start).
|
||||
*
|
||||
* @param {import('mysql2/promise').Pool | null | undefined} pool
|
||||
* @param {string} userId
|
||||
* @param {string} [projection]
|
||||
*/
|
||||
export async function loadSessionUserModelSnapshot(pool, userId, projection = 'default') {
|
||||
if (!pool?.query || !userId) return null;
|
||||
const canonicalUserId = resolveCanonicalUserId(userId);
|
||||
const snap = await getActiveSnapshot(pool, canonicalUserId, projection);
|
||||
if (!snap) return null;
|
||||
return {
|
||||
snapshot_id: snap.snapshot_id,
|
||||
user_id: snap.user_id,
|
||||
projection: snap.projection,
|
||||
profile_version: snap.profile_version,
|
||||
fast_revision: snap.fast_revision,
|
||||
content_hash: snap.content_hash,
|
||||
byte_size: snap.byte_size,
|
||||
stale_after_sec: snap.stale_after_sec,
|
||||
loaded_at_hint: 'session_start',
|
||||
core: snap.core,
|
||||
meta: snap.meta,
|
||||
...(canonicalUserId !== userId ? { canonical_user_id: canonicalUserId } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
const STOPWORDS = new Set([
|
||||
'的', '了', '在', '是', '我', '你', '他', '她', '它', '我们', '你们', '他们',
|
||||
'这', '那', '有', '和', '与', '或', '就', '也', '都', '还', '要', '会', '能',
|
||||
'一个', '什么', '怎么', '可以', '没有', '不是', '如果', '因为', '所以', '但是',
|
||||
'然后', '已经', '还是', '自己', '现在', '今天', '明天', '这个', '那个', '一下',
|
||||
'the', 'and', 'for', 'with', 'this', 'that', 'from', 'have', 'are', 'was', 'not',
|
||||
]);
|
||||
|
||||
function newId() {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
|
||||
function nowMs() {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function extractTerms(text) {
|
||||
const terms = [];
|
||||
const cjk = String(text).match(/[\u4e00-\u9fff]{2,12}/g) ?? [];
|
||||
terms.push(...cjk);
|
||||
const en = String(text).match(/[a-zA-Z][a-zA-Z0-9]{2,}/g) ?? [];
|
||||
terms.push(...en.map((w) => w.toLowerCase()));
|
||||
return terms.filter((t) => !STOPWORDS.has(t));
|
||||
}
|
||||
|
||||
function dayStartUtc(date) {
|
||||
const d = new Date(date);
|
||||
return new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()));
|
||||
}
|
||||
|
||||
function dayEndUtc(date) {
|
||||
const start = dayStartUtc(date);
|
||||
return new Date(start.getTime() + 24 * 60 * 60 * 1000 - 1);
|
||||
}
|
||||
|
||||
function hashSignal(userId, signalType, dimensionKey, windowStart, windowEnd, valueJson) {
|
||||
const raw = `${userId}|${signalType}|${dimensionKey}|${windowStart}|${windowEnd}|${JSON.stringify(valueJson)}`;
|
||||
return crypto.createHash('sha256').update(raw).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {object} envelope Evidence Envelope v1
|
||||
* @returns {Array<object>} signal drafts
|
||||
*/
|
||||
export function extractSignalsFromEnvelope(envelope) {
|
||||
if (envelope.evidence_type !== 'expression_segment') return [];
|
||||
const payload = envelope.payload ?? {};
|
||||
const text = payload.text ?? '';
|
||||
const occurredAt = envelope.occurred_at;
|
||||
const windowStart = dayStartUtc(occurredAt).toISOString().slice(0, 23).replace('T', ' ');
|
||||
const windowEnd = dayEndUtc(occurredAt).toISOString().slice(0, 23).replace('T', ' ');
|
||||
const signals = [];
|
||||
|
||||
for (const term of extractTerms(text)) {
|
||||
const key = /^[a-z]/.test(term) ? `term:${term}` : `term:${term}`;
|
||||
signals.push({
|
||||
signal_type: 'term_frequency',
|
||||
dimension_key: key,
|
||||
window_start: windowStart,
|
||||
window_end: windowEnd,
|
||||
value_json: { count: 1, chars: text.length },
|
||||
evidence_ids: [envelope.evidence_id],
|
||||
});
|
||||
}
|
||||
|
||||
const appId = payload.context?.app_bundle_id;
|
||||
if (appId) {
|
||||
signals.push({
|
||||
signal_type: 'app_usage',
|
||||
dimension_key: `app:${appId}`,
|
||||
window_start: windowStart,
|
||||
window_end: windowEnd,
|
||||
value_json: { count: 1, app_name: payload.context?.app ?? null },
|
||||
evidence_ids: [envelope.evidence_id],
|
||||
});
|
||||
}
|
||||
|
||||
const hour = new Date(occurredAt).getUTCHours();
|
||||
signals.push({
|
||||
signal_type: 'segment_count',
|
||||
dimension_key: `window:daily:${windowStart.slice(0, 10)}`,
|
||||
window_start: windowStart,
|
||||
window_end: windowEnd,
|
||||
value_json: { segments: 1, hour },
|
||||
evidence_ids: [envelope.evidence_id],
|
||||
});
|
||||
|
||||
return signals;
|
||||
}
|
||||
|
||||
export async function upsertSignals(pool, userId, signalDrafts) {
|
||||
let touched = 0;
|
||||
const ts = nowMs();
|
||||
for (const draft of signalDrafts) {
|
||||
const contentHash = hashSignal(
|
||||
userId,
|
||||
draft.signal_type,
|
||||
draft.dimension_key,
|
||||
draft.window_start,
|
||||
draft.window_end,
|
||||
draft.value_json,
|
||||
);
|
||||
const [existing] = await pool.query(
|
||||
`SELECT signal_id, value_json, evidence_ids FROM um_signals
|
||||
WHERE user_id = ? AND signal_type = ? AND dimension_key = ?
|
||||
AND window_start = ? AND window_end = ?
|
||||
LIMIT 1`,
|
||||
[userId, draft.signal_type, draft.dimension_key, draft.window_start, draft.window_end],
|
||||
);
|
||||
if (existing[0]) {
|
||||
const prev = existing[0];
|
||||
const prevValue = typeof prev.value_json === 'string' ? JSON.parse(prev.value_json) : prev.value_json;
|
||||
const prevEvidence =
|
||||
typeof prev.evidence_ids === 'string' ? JSON.parse(prev.evidence_ids) : prev.evidence_ids;
|
||||
const mergedEvidence = [...new Set([...(prevEvidence ?? []), ...draft.evidence_ids])];
|
||||
const mergedValue = {
|
||||
...prevValue,
|
||||
count: Number(prevValue.count ?? 0) + Number(draft.value_json.count ?? 1),
|
||||
segments: Number(prevValue.segments ?? 0) + Number(draft.value_json.segments ?? 0),
|
||||
chars: Number(prevValue.chars ?? 0) + Number(draft.value_json.chars ?? 0),
|
||||
};
|
||||
await pool.query(
|
||||
`UPDATE um_signals SET value_json = ?, evidence_ids = ?, computed_at = ?, content_hash = ?
|
||||
WHERE signal_id = ?`,
|
||||
[JSON.stringify(mergedValue), JSON.stringify(mergedEvidence), ts, contentHash, prev.signal_id],
|
||||
);
|
||||
} else {
|
||||
await pool.query(
|
||||
`INSERT INTO um_signals
|
||||
(signal_id, user_id, signal_type, dimension_key, window_start, window_end,
|
||||
value_json, evidence_ids, computed_at, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
[
|
||||
newId(),
|
||||
userId,
|
||||
draft.signal_type,
|
||||
draft.dimension_key,
|
||||
draft.window_start,
|
||||
draft.window_end,
|
||||
JSON.stringify(draft.value_json),
|
||||
JSON.stringify(draft.evidence_ids),
|
||||
ts,
|
||||
contentHash,
|
||||
],
|
||||
);
|
||||
}
|
||||
touched += 1;
|
||||
}
|
||||
return touched;
|
||||
}
|
||||
|
||||
export { extractTerms, STOPWORDS };
|
||||
@@ -0,0 +1,205 @@
|
||||
import crypto from 'node:crypto';
|
||||
|
||||
function newId() {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
|
||||
function nowMs() {
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
function hashJson(obj) {
|
||||
return crypto.createHash('sha256').update(JSON.stringify(obj)).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {import('mysql2/promise').Pool} pool
|
||||
* @param {string} userId
|
||||
* @param {object} options
|
||||
*/
|
||||
export async function materializeProfileAndSnapshot(pool, userId, options = {}) {
|
||||
const projection = options.projection ?? 'default';
|
||||
const reason = options.reason ?? 'candidate_accepted';
|
||||
const ts = nowMs();
|
||||
|
||||
const [projects] = await pool.query(
|
||||
`SELECT e.canonical_name, a.confidence, a.effective_weight, a.last_seen_at
|
||||
FROM um_entities e
|
||||
JOIN um_attributes a ON a.entity_id = e.entity_id AND a.status = 'active'
|
||||
WHERE e.user_id = ? AND e.entity_type = 'project' AND e.status = 'active'
|
||||
ORDER BY a.effective_weight DESC
|
||||
LIMIT 8`,
|
||||
[userId],
|
||||
);
|
||||
|
||||
const [focusCandidates] = await pool.query(
|
||||
`SELECT hypothesis_json, confidence, promotion_score, last_seen_at
|
||||
FROM um_candidates
|
||||
WHERE user_id = ? AND candidate_type = 'focus' AND status IN ('open', 'accepted')
|
||||
ORDER BY promotion_score DESC
|
||||
LIMIT 10`,
|
||||
[userId],
|
||||
);
|
||||
|
||||
const [versionRows] = await pool.query(
|
||||
`SELECT COALESCE(MAX(profile_version), 0) AS v FROM um_profile_versions WHERE user_id = ?`,
|
||||
[userId],
|
||||
);
|
||||
const profileVersion = Number(versionRows[0]?.v ?? 0) + 1;
|
||||
|
||||
const activeProjects = projects.map((row) => ({
|
||||
id: `project_${String(row.canonical_name).toLowerCase().replace(/[^a-z0-9]+/g, '_')}`,
|
||||
name: row.canonical_name,
|
||||
status: 'active',
|
||||
confidence: Number(row.confidence),
|
||||
last_seen: row.last_seen_at,
|
||||
}));
|
||||
|
||||
const recentFocus = focusCandidates.map((row) => {
|
||||
const h = typeof row.hypothesis_json === 'string' ? JSON.parse(row.hypothesis_json) : row.hypothesis_json;
|
||||
return {
|
||||
topic: h.topic ?? h.name ?? 'unknown',
|
||||
weight: Number(row.promotion_score),
|
||||
ttl_days: 14,
|
||||
};
|
||||
});
|
||||
|
||||
const structured = {
|
||||
identity: [],
|
||||
active_projects: activeProjects,
|
||||
recent_focus: recentFocus,
|
||||
technical_preferences: [],
|
||||
working_style: [],
|
||||
};
|
||||
|
||||
const structuredHash = hashJson(structured);
|
||||
await pool.query(
|
||||
`UPDATE um_profile_versions SET status = 'superseded' WHERE user_id = ? AND status = 'active'`,
|
||||
[userId],
|
||||
);
|
||||
await pool.query(
|
||||
`INSERT INTO um_profile_versions
|
||||
(profile_version, user_id, structured_json, content_hash, parent_version, materialize_reason, created_at, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'active')`,
|
||||
[
|
||||
profileVersion,
|
||||
userId,
|
||||
JSON.stringify(structured),
|
||||
structuredHash,
|
||||
profileVersion > 1 ? profileVersion - 1 : null,
|
||||
reason,
|
||||
ts,
|
||||
],
|
||||
);
|
||||
|
||||
const [fastRows] = await pool.query(
|
||||
`SELECT COALESCE(MAX(fast_revision), 0) AS r FROM um_profile_snapshots WHERE user_id = ? AND projection = ?`,
|
||||
[userId, projection],
|
||||
);
|
||||
const fastRevision = Number(fastRows[0]?.r ?? 0) + 1;
|
||||
|
||||
const agentHints = [];
|
||||
if (activeProjects[0]) agentHints.push(`近期重点:${activeProjects[0].name}`);
|
||||
if (recentFocus[0]) agentHints.push(`关注话题:${recentFocus[0].topic}`);
|
||||
|
||||
const snapshotCore = {
|
||||
identity: structured.identity,
|
||||
active_projects: activeProjects,
|
||||
recent_focus: recentFocus,
|
||||
technical_preferences: structured.technical_preferences,
|
||||
working_style: structured.working_style,
|
||||
agent_hints: agentHints,
|
||||
};
|
||||
|
||||
const snapshotBody = {
|
||||
profile_version: profileVersion,
|
||||
fast_revision: fastRevision,
|
||||
projection,
|
||||
core: snapshotCore,
|
||||
meta: {
|
||||
graph_entity_count: activeProjects.length,
|
||||
open_candidates: recentFocus.length,
|
||||
},
|
||||
};
|
||||
|
||||
const snapshotJson = {
|
||||
...snapshotBody,
|
||||
stale_after_sec: 3600,
|
||||
};
|
||||
const snapshotStr = JSON.stringify(snapshotCore);
|
||||
const byteSize = Buffer.byteLength(snapshotStr, 'utf8');
|
||||
const contentHash = hashJson(snapshotBody);
|
||||
|
||||
await pool.query(
|
||||
`UPDATE um_profile_snapshots SET status = 'superseded' WHERE user_id = ? AND projection = ? AND status = 'active'`,
|
||||
[userId, projection],
|
||||
);
|
||||
|
||||
const snapshotId = newId();
|
||||
await pool.query(
|
||||
`INSERT INTO um_profile_snapshots
|
||||
(snapshot_id, user_id, projection, profile_version, fast_revision, snapshot_json, byte_size,
|
||||
content_hash, created_at, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'active')`,
|
||||
[
|
||||
snapshotId,
|
||||
userId,
|
||||
projection,
|
||||
profileVersion,
|
||||
fastRevision,
|
||||
JSON.stringify(snapshotJson),
|
||||
byteSize,
|
||||
contentHash,
|
||||
ts,
|
||||
],
|
||||
);
|
||||
|
||||
return {
|
||||
profile_version: profileVersion,
|
||||
fast_revision: fastRevision,
|
||||
snapshot_id: snapshotId,
|
||||
content_hash: contentHash,
|
||||
byte_size: byteSize,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getActiveSnapshot(pool, userId, projection = 'default') {
|
||||
const [rows] = await pool.query(
|
||||
`SELECT snapshot_id, user_id, projection, profile_version, fast_revision, snapshot_json,
|
||||
byte_size, content_hash, created_at
|
||||
FROM um_profile_snapshots
|
||||
WHERE user_id = ? AND projection = ? AND status = 'active'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1`,
|
||||
[userId, projection],
|
||||
);
|
||||
const row = rows[0];
|
||||
if (!row) return null;
|
||||
const snapshotJson =
|
||||
typeof row.snapshot_json === 'string' ? JSON.parse(row.snapshot_json) : row.snapshot_json;
|
||||
return {
|
||||
snapshot_id: row.snapshot_id,
|
||||
user_id: row.user_id,
|
||||
projection: row.projection,
|
||||
profile_version: row.profile_version,
|
||||
fast_revision: row.fast_revision,
|
||||
content_hash: row.content_hash,
|
||||
byte_size: row.byte_size,
|
||||
stale_after_sec: snapshotJson.stale_after_sec ?? 3600,
|
||||
core: snapshotJson.core ?? snapshotJson,
|
||||
meta: snapshotJson.meta ?? {},
|
||||
created_at: row.created_at,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getSnapshotInfo(pool, userId, projection = 'default') {
|
||||
const snap = await getActiveSnapshot(pool, userId, projection);
|
||||
if (!snap) return null;
|
||||
return {
|
||||
profile_version: snap.profile_version,
|
||||
fast_revision: snap.fast_revision,
|
||||
content_hash: snap.content_hash,
|
||||
projection: snap.projection,
|
||||
snapshot_id: snap.snapshot_id,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user