john
50 lines
1.5 KiB
JavaScript
50 lines
1.5 KiB
JavaScript
const DEFAULT_DIMENSIONS = 3;
|
|
|
|
function resolveDimensions(value) {
|
|
const dimensions = Number(value ?? process.env.MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS ?? DEFAULT_DIMENSIONS);
|
|
if (!Number.isInteger(dimensions) || dimensions < 1 || dimensions > 4096) {
|
|
throw new Error(`Invalid MEMORY_V2_LOCAL_EMBEDDING_DIMENSIONS: ${value}`);
|
|
}
|
|
return dimensions;
|
|
}
|
|
|
|
function hashToken(token) {
|
|
let hash = 2166136261;
|
|
for (const char of String(token)) {
|
|
hash ^= char.codePointAt(0);
|
|
hash = Math.imul(hash, 16777619);
|
|
}
|
|
return hash >>> 0;
|
|
}
|
|
|
|
function tokenize(text) {
|
|
return String(text ?? '')
|
|
.toLowerCase()
|
|
.replace(/[^a-z0-9\u4e00-\u9fa5]+/gu, ' ')
|
|
.trim()
|
|
.split(/\s+/)
|
|
.filter(Boolean);
|
|
}
|
|
|
|
export function embedText(text, { dimensions = undefined } = {}) {
|
|
const resolvedDimensions = resolveDimensions(dimensions);
|
|
const vector = Array.from({ length: resolvedDimensions }, () => 0);
|
|
const tokens = tokenize(text);
|
|
for (const token of tokens.length ? tokens : ['empty']) {
|
|
const hash = hashToken(token);
|
|
const index = hash % resolvedDimensions;
|
|
const sign = hash & 1 ? 1 : -1;
|
|
vector[index] += sign * (1 + (token.length % 7));
|
|
}
|
|
const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1;
|
|
return vector.map((value) => Number((value / norm).toFixed(8)));
|
|
}
|
|
|
|
export async function embedQuery(query, input = {}) {
|
|
return embedText(query, {
|
|
dimensions: input.dimensions ?? input.embeddingDimensions,
|
|
});
|
|
}
|
|
|
|
export default embedQuery;
|