tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add pluggable search service registry

Browse Source
This commit is contained in:
john
2026-07-23 21:38:36 +08:00
parent 48e9f68211
commit 4e36b98c9e
7 changed files with 506 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mindsearch-config.mjs
+288 -10
View File
@@ -1,11 +1,56 @@
const CONFIG_TABLE = 'h5_mindsearch_config';
const CONFIG_SCOPE = 'global';
const SERVICE_KINDS = new Set(['provider', 'reader', 'orchestrator']);
const SERVICE_ADAPTERS = new Set(['searxng', 'github', 'reader', 'research-http']);
const ROUTE_KEYS = ['web', 'news', 'code', 'read', 'research'];
const BUILTIN_SERVICES = Object.freeze([
{
id: 'searxng',
name: 'SearXNG',
kind: 'provider',
adapter: 'searxng',
capabilities: ['search.web', 'search.news'],
endpoint: '',
healthPath: '/config',
enabled: false,
timeoutMs: 8000,
priority: 100,
},
{
id: 'github',
name: 'GitHub Code',
kind: 'provider',
adapter: 'github',
capabilities: ['search.code'],
endpoint: 'https://api.github.com/search/code',
healthPath: '',
enabled: false,
timeoutMs: 8000,
priority: 90,
},
{
id: 'reader',
name: 'Safe Reader',
kind: 'reader',
adapter: 'reader',
capabilities: ['search.read'],
endpoint: '',
healthPath: '',
enabled: false,
timeoutMs: 8000,
priority: 80,
},
]);
export const MINDSEARCH_DEFAULT_CONFIG = Object.freeze({
enabled: false,
mode: 'off',
providers: { searxng: false, github: false, reader: false },
settings: { searxngEndpoint: '', maxResults: 10, timeoutMs: 8000, readerMaxChars: 12000 },
services: BUILTIN_SERVICES,
routes: { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: '' },
});
const clone = (value) => JSON.parse(JSON.stringify(value));
@@ -15,22 +60,182 @@ const bool = (value, fallback = false) => {
return /^(1|true|yes|on)$/i.test(String(value));
};
const boundedNumber = (value, fallback, min, max) =>
Math.max(min, Math.min(max, Number(value ?? fallback) || fallback));
export function isValidSearchServiceEndpoint(value, { allowEmpty = true } = {}) {
const endpoint = String(value ?? '').trim();
if (!endpoint) return allowEmpty;
try {
const url = new URL(endpoint);
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) return false;
return !['0.0.0.0', '::', '[::]', '169.254.169.254'].includes(url.hostname);
} catch {
return false;
}
}
function normalizeCapabilities(value, fallback = []) {
const capabilities = Array.isArray(value) ? value : fallback;
return [...new Set(capabilities
.map((item) => String(item ?? '').trim())
.filter((item) => /^[a-z][a-z0-9_.-]{1,63}$/.test(item)))]
.slice(0, 20);
}
function normalizeService(input = {}, fallback = {}) {
const id = String(input.id ?? fallback.id ?? '').trim().toLowerCase();
if (!/^[a-z0-9][a-z0-9._-]{0,63}$/.test(id)) return null;
const kind = SERVICE_KINDS.has(input.kind) ? input.kind : (fallback.kind ?? 'provider');
const adapter = SERVICE_ADAPTERS.has(input.adapter) ? input.adapter : (fallback.adapter ?? 'searxng');
const endpoint = String(input.endpoint ?? fallback.endpoint ?? '').trim();
if (!isValidSearchServiceEndpoint(endpoint)) return null;
const healthPathValue = String(input.healthPath ?? fallback.healthPath ?? '').trim();
const healthPath = healthPathValue && /^\/[^\s]{0,255}$/.test(healthPathValue) ? healthPathValue : '';
return {
id,
name: String(input.name ?? fallback.name ?? id).trim().slice(0, 80) || id,
kind,
adapter,
capabilities: normalizeCapabilities(input.capabilities, fallback.capabilities),
endpoint,
healthPath,
enabled: bool(input.enabled, bool(fallback.enabled)),
timeoutMs: boundedNumber(input.timeoutMs, fallback.timeoutMs ?? 8000, 1000, 120000),
priority: boundedNumber(input.priority, fallback.priority ?? 100, 1, 1000),
};
}
function builtinsFromLegacy(input = {}) {
const providers = input.providers ?? {};
const settings = input.settings ?? {};
return BUILTIN_SERVICES.map((service) => normalizeService({
...service,
enabled: bool(providers[service.adapter], service.enabled),
endpoint: service.adapter === 'searxng'
? String(settings.searxngEndpoint ?? service.endpoint)
: service.endpoint,
timeoutMs: settings.timeoutMs ?? service.timeoutMs,
}, service));
}
function normalizeServices(input = {}) {
const source = Array.isArray(input.services) ? input.services : builtinsFromLegacy(input);
const seen = new Set();
const services = [];
for (const item of source) {
const fallback = BUILTIN_SERVICES.find((candidate) => candidate.id === item?.id) ?? {};
const service = normalizeService(item, fallback);
if (!service || seen.has(service.id)) continue;
seen.add(service.id);
services.push(service);
}
for (const builtin of builtinsFromLegacy(input)) {
if (!seen.has(builtin.id)) services.push(builtin);
}
return services.slice(0, 30);
}
function normalizeRoutes(value, services) {
const ids = new Set(services.map((service) => service.id));
const defaults = MINDSEARCH_DEFAULT_CONFIG.routes;
return Object.fromEntries(ROUTE_KEYS.map((key) => {
const requested = String(value?.[key] ?? defaults[key] ?? '').trim();
return [key, !requested || ids.has(requested) ? requested : ''];
}));
}
export function resolveMindSearchService(config, route) {
const normalized = normalizeMindSearchConfig(config);
const serviceId = normalized.routes?.[route] ?? '';
return normalized.services.find((service) => service.id === serviceId && service.enabled) ?? null;
}
export async function probeMindSearchService(service, { fetchImpl = fetch } = {}) {
const normalized = normalizeService(service);
if (!normalized) throw new Error('Invalid search service configuration');
if (!normalized.endpoint) {
return {
ok: normalized.adapter === 'reader',
serviceId: normalized.id,
adapter: normalized.adapter,
status: null,
latencyMs: 0,
message: normalized.adapter === 'reader' ? 'Built-in reader is available' : 'Service endpoint is not configured',
};
}
const startedAt = Date.now();
let url;
if (normalized.adapter === 'searxng') {
url = new URL(normalized.endpoint);
url.searchParams.set('q', 'OpenAI');
url.searchParams.set('format', 'json');
} else if (normalized.healthPath) {
url = new URL(normalized.healthPath, normalized.endpoint);
} else {
return {
ok: true,
serviceId: normalized.id,
adapter: normalized.adapter,
status: null,
latencyMs: 0,
message: 'Configuration is valid; no health path is configured',
};
}
try {
const response = await fetchImpl(url, {
signal: AbortSignal.timeout(normalized.timeoutMs),
headers: { accept: 'application/json,text/plain' },
});
let resultCount = null;
if (normalized.adapter === 'searxng' && response.ok) {
const body = await response.json();
resultCount = Array.isArray(body.results) ? body.results.length : 0;
}
return {
ok: response.ok && (resultCount == null || resultCount > 0),
serviceId: normalized.id,
adapter: normalized.adapter,
status: response.status,
latencyMs: Date.now() - startedAt,
resultCount,
message: response.ok ? 'Health check completed' : `Service returned HTTP ${response.status}`,
};
} catch (error) {
return {
ok: false,
serviceId: normalized.id,
adapter: normalized.adapter,
status: null,
latencyMs: Date.now() - startedAt,
message: error instanceof Error ? error.message : 'Service health check failed',
};
}
}
export function normalizeMindSearchConfig(input = {}) {
const mode = ['off', 'shadow', 'assist'].includes(input.mode) ? input.mode : 'off';
const services = normalizeServices(input);
const serviceByAdapter = (adapter) => services.find((service) => service.adapter === adapter);
const searxng = serviceByAdapter('searxng');
const github = serviceByAdapter('github');
const reader = serviceByAdapter('reader');
const config = {
enabled: bool(input.enabled),
mode,
providers: {
searxng: bool(input.providers?.searxng),
github: bool(input.providers?.github),
reader: bool(input.providers?.reader),
searxng: bool(searxng?.enabled),
github: bool(github?.enabled),
reader: bool(reader?.enabled),
},
settings: {
searxngEndpoint: String(input.settings?.searxngEndpoint ?? '').trim(),
maxResults: Math.max(1, Math.min(20, Number(input.settings?.maxResults ?? 10) || 10)),
timeoutMs: Math.max(1000, Math.min(30000, Number(input.settings?.timeoutMs ?? 8000) || 8000)),
readerMaxChars: Math.max(1000, Math.min(50000, Number(input.settings?.readerMaxChars ?? 12000) || 12000)),
searxngEndpoint: String(searxng?.endpoint ?? input.settings?.searxngEndpoint ?? '').trim(),
maxResults: boundedNumber(input.settings?.maxResults, 10, 1, 20),
timeoutMs: boundedNumber(input.settings?.timeoutMs, 8000, 1000, 30000),
readerMaxChars: boundedNumber(input.settings?.readerMaxChars, 12000, 1000, 50000),
},
services,
routes: normalizeRoutes(input.routes, services),
};
if (!config.enabled || config.mode === 'off') {
config.enabled = false;
@@ -49,6 +254,57 @@ function envConfig(env = process.env) {
github: env.TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED,
reader: env.TKMIND_SEARCH_READER_ENABLED,
},
settings: {
searxngEndpoint: env.TKMIND_SEARCH_SEARXNG_URL,
maxResults: env.TKMIND_SEARCH_MAX_RESULTS,
timeoutMs: env.TKMIND_SEARCH_TIMEOUT_MS,
readerMaxChars: env.TKMIND_SEARCH_READER_MAX_CHARS,
},
services: parseJsonArray(env.TKMIND_SEARCH_SERVICES_JSON),
routes: parseJsonObject(env.TKMIND_SEARCH_ROUTES_JSON),
});
}
function parseJsonArray(value) {
if (!value) return undefined;
try {
const parsed = JSON.parse(value);
return Array.isArray(parsed) ? parsed : undefined;
} catch {
return undefined;
}
}
function parseJsonObject(value) {
if (!value) return undefined;
try {
const parsed = JSON.parse(value);
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : undefined;
} catch {
return undefined;
}
}
function applyLegacyPatchToServices(current, patch) {
if (Array.isArray(patch?.services)) return patch.services;
const providerPatch = patch?.providers ?? {};
const settingPatch = patch?.settings ?? {};
return current.services.map((service) => {
if (service.adapter === 'searxng') {
return {
...service,
...(providerPatch.searxng == null ? {} : { enabled: providerPatch.searxng }),
...(settingPatch.searxngEndpoint == null ? {} : { endpoint: settingPatch.searxngEndpoint }),
...(settingPatch.timeoutMs == null ? {} : { timeoutMs: settingPatch.timeoutMs }),
};
}
if (service.adapter === 'github' && providerPatch.github != null) {
return { ...service, enabled: providerPatch.github };
}
if (service.adapter === 'reader' && providerPatch.reader != null) {
return { ...service, enabled: providerPatch.reader };
}
return service;
});
}
@@ -61,7 +317,7 @@ export async function ensureMindSearchConfigSchema(pool) {
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`);
}
export function createMindSearchConfigService(pool, { env = process.env } = {}) {
export function createMindSearchConfigService(pool, { env = process.env, fetchImpl = fetch } = {}) {
let ready = null;
const ensure = async () => {
if (!pool) return;
@@ -82,9 +338,22 @@ export function createMindSearchConfigService(pool, { env = process.env } = {})
ensureSchema: ensure,
getEffectiveConfig: async () => (await load()).config,
getAdminConfig: async () => load(),
testService: async (serviceId) => {
const current = await load();
const service = current.config.services.find((item) => item.id === String(serviceId ?? '').trim());
if (!service) throw Object.assign(new Error('Search service not found'), { code: 'SEARCH_SERVICE_NOT_FOUND' });
return probeMindSearchService(service, { fetchImpl });
},
updateAdminConfig: async (patch, { updatedBy = null } = {}) => {
const current = await load();
const next = normalizeMindSearchConfig({ ...current.config, ...patch, providers: { ...current.config.providers, ...(patch?.providers ?? {}) }, settings: { ...current.config.settings, ...(patch?.settings ?? {}) } });
const next = normalizeMindSearchConfig({
...current.config,
...patch,
providers: { ...current.config.providers, ...(patch?.providers ?? {}) },
settings: { ...current.config.settings, ...(patch?.settings ?? {}) },
services: applyLegacyPatchToServices(current.config, patch),
routes: { ...current.config.routes, ...(patch?.routes ?? {}) },
});
if (!pool) return { config: next, source: 'env' };
await ensure();
const now = Date.now();
@@ -93,7 +362,16 @@ export function createMindSearchConfigService(pool, { env = process.env } = {})
},
getRuntimeState: async () => {
const config = await (async () => (await load()).config)();
return { enabled: config.enabled, mode: config.mode, providers: config.providers, effective: config.enabled && config.mode !== 'off' };
return {
enabled: config.enabled,
mode: config.mode,
providers: config.providers,
services: config.services.map(({ id, name, kind, adapter, capabilities, enabled, timeoutMs, priority }) => ({
id, name, kind, adapter, capabilities, enabled, timeoutMs, priority,
})),
routes: config.routes,
effective: config.enabled && config.mode !== 'off',
};
},
};
}