diff --git a/.env.example b/.env.example index 4b8df20..912b13a 100644 --- a/.env.example +++ b/.env.example @@ -270,6 +270,24 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind # Sandbox MCP 调用 Portal 内部生图入口；容器内通常使用 host.docker.internal。 # MINDSPACE_AGENT_API_BASE_URL=http://127.0.0.1:8081/api +# --------------------------------------------------------------------------- +# MindSearch / standalone Deep Search +# --------------------------------------------------------------------------- +# Production SearXNG on 103 is bound to the host at 127.0.0.1:20080. +# Goosed MCP endpoints are automatically rewritten to host.docker.internal. +# TKMIND_SEARCH_SEARXNG_URL=http://127.0.0.1:20080/search +# TKMIND_SEARCH_MCP_HOST_GATEWAY=host.docker.internal +# +# Deep Search runs as an independently released LaunchAgent on 127.0.0.1:20100. +# Its production environment is stored separately at: +# /Users/john/Project/deep-search-runtime/shared/.env +# TKMIND_DEEP_SEARCH_SECRET= +# TKMIND_SEARCH_GITHUB_GATEWAY_URL=http://127.0.0.1:20100 +# TKMIND_SEARCH_GITHUB_GATEWAY_SECRET=${TKMIND_DEEP_SEARCH_SECRET} +# +# GITHUB_TOKEN must only be set in the standalone Deep Search environment. +# Do not pass it to goosed MCP extension envs or commit it to Git. + # --------------------------------------------------------------------------- # Memind runtime profile（本地 vs 生产 必须区分） # 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。 diff --git a/.gitignore b/.gitignore index 5599414..629cc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ops/node_modules/ dist/ ops/dist/ .runtime/ +.deep-search/ .env .env.local diff --git a/admin-routes.mjs b/admin-routes.mjs index 0a9b58f..debe42d 100644 --- a/admin-routes.mjs +++ b/admin-routes.mjs @@ -207,6 +207,15 @@ export function createAdminApi({ if (!mindSearchConfigService?.getRuntimeState) return res.status(503).json({ message: 'MindSearch 配置服务未启用' }); return res.json(await mindSearchConfigService.getRuntimeState()); }); + adminApi.post('/mindsearch/services/:serviceId/test', requireAdmin, async (req, res) => { + if (!mindSearchConfigService?.testService) return res.status(503).json({ message: 'MindSearch 服务测试未启用' }); + try { + return res.json(await mindSearchConfigService.testService(req.params.serviceId)); + } catch (error) { + if (error?.code === 'SEARCH_SERVICE_NOT_FOUND') return res.status(404).json({ message: error.message }); + throw error; + } + }); adminApi.get('/memory-v2/runtime', requireAdmin, async (_req, res) => { if (!memoryV2ConfigService?.getRuntimeState) { diff --git a/admin-routes.test.mjs b/admin-routes.test.mjs index c6b8f3b..18c0b90 100644 --- a/admin-routes.test.mjs +++ b/admin-routes.test.mjs @@ -165,6 +165,7 @@ test('admin system disclosure policy routes version config through the injected test('admin MindSearch routes persist only through injected control-plane service', async () => { const updates = []; + const testedServices = []; const router = createAdminApi({ jsonBody: express.json(), getToken() { return 'token-admin'; }, userAuth: { async getMe() { return { id: 'admin-1', role: 'admin' }; } }, @@ -173,6 +174,7 @@ test('admin MindSearch routes persist only through injected control-plane servic async getAdminConfig() { return { config: { enabled: false, mode: 'off', providers: { searxng: false, github: false, reader: false } }, source: 'env' }; }, async updateAdminConfig(patch, context) { updates.push({ patch, context }); return { config: { enabled: true, mode: 'shadow', providers: { searxng: true, github: false, reader: false } }, source: 'admin' }; }, async getRuntimeState() { return { effective: false, mode: 'off' }; }, + async testService(serviceId) { testedServices.push(serviceId); return { ok: true, serviceId, latencyMs: 3 }; }, }, plazaPosts: null, plazaOps: null, wechatAdmin: null, subscriptionService: null, }); @@ -185,6 +187,9 @@ test('admin MindSearch routes persist only through injected control-plane servic assert.deepEqual(updates, [{ patch: { enabled: true, mode: 'shadow' }, context: { updatedBy: 'admin-1' } }]); const runtime = await fetch(`${server.baseUrl}/admin-api/mindsearch/runtime`, { headers: { cookie: 'h5_user_session=token-admin' } }); assert.deepEqual(await runtime.json(), { effective: false, mode: 'off' }); + const serviceTest = await fetch(`${server.baseUrl}/admin-api/mindsearch/services/searxng/test`, { method: 'POST', headers: { cookie: 'h5_user_session=token-admin' } }); + assert.deepEqual(await serviceTest.json(), { ok: true, serviceId: 'searxng', latencyMs: 3 }); + assert.deepEqual(testedServices, ['searxng']); } finally { await server.close(); } }); diff --git a/capabilities.mjs b/capabilities.mjs index 187a91f..3b26844 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -48,6 +48,23 @@ export function resolveMindSearchMcpServerPath(overridePath) { return path.join(path.dirname(fileURLToPath(import.meta.url)), 'tkmind-search-mcp.mjs'); } +export function resolveMindSearchMcpEndpoint( + endpoint, + { hostGateway = process.env.TKMIND_SEARCH_MCP_HOST_GATEWAY || 'host.docker.internal' } = {}, +) { + const source = String(endpoint ?? '').trim(); + if (!source) return source; + try { + const parsed = new URL(source); + if (LOOPBACK_PG_HOSTS.has(parsed.hostname)) { + parsed.hostname = String(hostGateway || 'host.docker.internal').trim(); + } + return parsed.toString(); + } catch { + return source; + } +} + export function resolveExcelMcpServerPath(overridePath) { const normalized = String(overridePath ?? '').trim(); if (normalized) return normalized; @@ -401,7 +418,14 @@ function sandboxMcpEnvs(sandboxMcp, mcpTools) { */ export function buildAgentExtensionPolicy( capabilities, - { unrestricted = false, policies = null, sandboxMcp = null, toolMode = 'chat', mindSearchConfig = null } = {}, + { + unrestricted = false, + policies = null, + sandboxMcp = null, + toolMode = 'chat', + mindSearchConfig = null, + userId = null, + } = {}, ) { if (unrestricted) { return { extensionOverrides: null, enableContextMemory: true, gooseMode: 'auto' }; @@ -496,7 +520,59 @@ export function buildAgentExtensionPolicy( } const searchEnabled = capabilities.search_external && mindSearchConfig?.enabled && mindSearchConfig.mode !== 'off'; if (searchEnabled && (!policies || policies.network_egress !== 'deny')) { - extensions.push({ type: 'stdio', name: 'tkmind-search', description: 'MindSearch 外部搜索增强（补充能力）', display_name: 'tkmind-search', bundled: false, cmd: resolveSandboxMcpNodeExecPath(process.env.GOOSED_MCP_NODE_PATH), args: [resolveMindSearchMcpServerPath(process.env.TKMIND_SEARCH_MCP_SERVER_PATH)], envs: { TKMIND_SEARCH_ENABLED: '1', TKMIND_SEARCH_MODE: mindSearchConfig.mode, TKMIND_SEARCH_PROVIDER_SEARXNG_ENABLED: mindSearchConfig.providers?.searxng ? '1' : '0', TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED: mindSearchConfig.providers?.github ? '1' : '0', TKMIND_SEARCH_READER_ENABLED: mindSearchConfig.providers?.reader ? '1' : '0', TKMIND_SEARCH_SEARXNG_URL: mindSearchConfig.settings?.searxngEndpoint ?? '', TKMIND_SEARCH_MAX_RESULTS: String(mindSearchConfig.settings?.maxResults ?? 10), TKMIND_SEARCH_TIMEOUT_MS: String(mindSearchConfig.settings?.timeoutMs ?? 8000), TKMIND_SEARCH_READER_MAX_CHARS: String(mindSearchConfig.settings?.readerMaxChars ?? 12000) }, available_tools: ['tkmind_search', 'tkmind_read'] }); + const mcpServices = (mindSearchConfig.services ?? []).map((service) => ({ + ...service, + endpoint: resolveMindSearchMcpEndpoint(service.endpoint), + })); + const researchService = mcpServices.find((service) => + service.id === mindSearchConfig.routes?.research && service.adapter === 'research-http'); + const deepSearchService = mcpServices.find((service) => + service.id === 'deep-search' && service.adapter === 'research-http'); + const hasResearchService = Boolean(researchService?.enabled); + const githubGatewayUrl = resolveMindSearchMcpEndpoint( + process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL + || deepSearchService?.endpoint + || researchService?.endpoint + || '', + ); + extensions.push({ + type: 'stdio', + name: 'tkmind-search', + description: 'MindSearch 外部搜索增强（补充能力）', + display_name: 'tkmind-search', + bundled: false, + cmd: resolveSandboxMcpNodeExecPath(process.env.GOOSED_MCP_NODE_PATH), + args: [resolveMindSearchMcpServerPath(process.env.TKMIND_SEARCH_MCP_SERVER_PATH)], + envs: { + TKMIND_SEARCH_ENABLED: '1', + TKMIND_SEARCH_MODE: mindSearchConfig.mode, + TKMIND_SEARCH_PROVIDER_SEARXNG_ENABLED: mindSearchConfig.providers?.searxng ? '1' : '0', + TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED: mindSearchConfig.providers?.github ? '1' : '0', + TKMIND_SEARCH_READER_ENABLED: mindSearchConfig.providers?.reader ? '1' : '0', + TKMIND_SEARCH_SEARXNG_URL: resolveMindSearchMcpEndpoint( + mindSearchConfig.settings?.searxngEndpoint ?? '', + ), + TKMIND_SEARCH_MAX_RESULTS: String(mindSearchConfig.settings?.maxResults ?? 10), + TKMIND_SEARCH_TIMEOUT_MS: String(mindSearchConfig.settings?.timeoutMs ?? 8000), + TKMIND_SEARCH_READER_MAX_CHARS: String(mindSearchConfig.settings?.readerMaxChars ?? 12000), + TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mcpServices), + TKMIND_SEARCH_ROUTES_JSON: JSON.stringify(mindSearchConfig.routes ?? {}), + TKMIND_SEARCH_USER_ID: userId ? String(userId) : '', + TKMIND_DEEP_SEARCH_SECRET: process.env.TKMIND_DEEP_SEARCH_SECRET ?? '', + TKMIND_SEARCH_GITHUB_GATEWAY_URL: githubGatewayUrl, + TKMIND_SEARCH_GITHUB_GATEWAY_SECRET: + process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET + ?? process.env.TKMIND_DEEP_SEARCH_SECRET + ?? '', + }, + available_tools: [ + 'tkmind_search', + 'tkmind_read', + ...(hasResearchService + ? ['tkmind_research', 'tkmind_research_status', 'tkmind_research_cancel'] + : []), + ], + }); } if (capabilities.excel_analysis) { const excelWorkspaceRoot = resolveSandboxMcpLocalRoot(sandboxMcp); diff --git a/capabilities.test.mjs b/capabilities.test.mjs index 84bf9cd..be36471 100644 --- a/capabilities.test.mjs +++ b/capabilities.test.mjs @@ -8,6 +8,7 @@ import { clampUserCapabilities, DEFAULT_USER_CAPABILITIES, normalizeCapabilityPatch, + resolveMindSearchMcpEndpoint, resolveSandboxMcpNodeExecPath, resolveSandboxMcpServerPath, resolveSandboxMcpUserDataPgUrl, @@ -53,6 +54,17 @@ test('resolveSandboxMcpUserDataPgUrl preserves native URLs and honors an explici ); }); +test('resolveMindSearchMcpEndpoint rewrites host loopback for goosed containers', () => { + assert.equal( + resolveMindSearchMcpEndpoint('http://127.0.0.1:20080/search'), + 'http://host.docker.internal:20080/search', + ); + assert.equal( + resolveMindSearchMcpEndpoint('https://search.internal/query'), + 'https://search.internal/query', + ); +}); + test('resolveSandboxMcpNodeExecPath honors container-path override without host fs checks', () => { assert.equal(resolveSandboxMcpNodeExecPath('/usr/local/bin/node'), '/usr/local/bin/node'); assert.equal(resolveSandboxMcpNodeExecPath(''), process.execPath); diff --git a/deep-search-engine.mjs b/deep-search-engine.mjs new file mode 100644 index 0000000..2ffdbcf --- /dev/null +++ b/deep-search-engine.mjs @@ -0,0 +1,802 @@ +import dns from 'node:dns/promises'; +import { EventEmitter } from 'node:events'; +import net from 'node:net'; +import { randomUUID } from 'node:crypto'; +import { searchSearxng } from './mindsearch-providers.mjs'; + +const DEPTH_PROFILES = Object.freeze({ + quick: { goals: 3, rounds: 2, resultsPerQuery: 6, maxSources: 8 }, + standard: { goals: 4, rounds: 2, resultsPerQuery: 7, maxSources: 14 }, + deep: { goals: 6, rounds: 3, resultsPerQuery: 10, maxSources: 24 }, +}); + +const TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|mc_)/i; + +function clamp(value, min, max) { + return Math.max(min, Math.min(max, value)); +} + +function throwIfAborted(signal) { + if (signal?.aborted) { + const error = new Error('Research task was cancelled'); + error.name = 'AbortError'; + throw error; + } +} + +function decodeHtml(value) { + return value + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'|'/gi, "'") + .replace(/—/gi, '—') + .replace(/–/gi, '–') + .replace(/…/gi, '…') + .replace(/&#(\d+);/g, (_match, code) => String.fromCodePoint(Number(code))); +} + +export function htmlToPlainText(html) { + return decodeHtml(String(html ?? '') + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/]*>[\s\S]*?<\/noscript>/gi, ' ') + .replace(//g, ' ') + .replace(/<\/(p|div|section|article|li|h[1-6]|tr)>/gi, '\n') + .replace(//gi, '\n') + .replace(/<[^>]+>/g, ' ')) + .replace(/[ \t]+/g, ' ') + .replace(/\n\s+/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +export function tokenize(value) { + const normalized = String(value ?? '').toLowerCase(); + const latin = normalized.match(/[a-z0-9][a-z0-9._-]{1,}/g) ?? []; + const cjkSequences = normalized.match(/[\p{Script=Han}]{2,}/gu) ?? []; + const cjk = []; + for (const sequence of cjkSequences) { + cjk.push(sequence); + for (let index = 0; index < sequence.length - 1; index += 1) { + cjk.push(sequence.slice(index, index + 2)); + } + } + return [...new Set([...latin, ...cjk])]; +} + +function lexicalScore(needle, haystack) { + const terms = tokenize(needle); + if (!terms.length) return 0; + const text = String(haystack ?? '').toLowerCase(); + return terms.filter((term) => text.includes(term)).length / terms.length; +} + +export function canonicalizeUrl(value) { + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol)) return ''; + url.hash = ''; + for (const key of [...url.searchParams.keys()]) { + if (TRACKING_PARAMS.test(key)) url.searchParams.delete(key); + } + url.hostname = url.hostname.toLowerCase(); + if ((url.protocol === 'https:' && url.port === '443') || (url.protocol === 'http:' && url.port === '80')) { + url.port = ''; + } + return url.toString(); + } catch { + return ''; + } +} + +function isPrivateIp(address) { + const normalized = String(address ?? '').toLowerCase(); + if (net.isIPv4(normalized)) { + const parts = normalized.split('.').map(Number); + return parts[0] === 0 + || parts[0] === 10 + || parts[0] === 127 + || (parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) + || (parts[0] === 169 && parts[1] === 254) + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168) + || parts[0] >= 224; + } + if (net.isIPv6(normalized)) { + return normalized === '::' + || normalized === '::1' + || normalized.startsWith('fc') + || normalized.startsWith('fd') + || normalized.startsWith('fe8') + || normalized.startsWith('fe9') + || normalized.startsWith('fea') + || normalized.startsWith('feb') + || normalized.startsWith('::ffff:127.') + || normalized.startsWith('::ffff:10.') + || normalized.startsWith('::ffff:192.168.'); + } + return false; +} + +export async function assertSafePublicUrl(value, { lookup = dns.lookup } = {}) { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + throw new Error('Unsafe source URL'); + } + if (url.hostname === 'localhost' || isPrivateIp(url.hostname)) throw new Error('Unsafe source URL'); + const records = await lookup(url.hostname, { all: true, verbatim: true }); + if (!records.length || records.some((record) => isPrivateIp(record.address))) { + throw new Error('Unsafe source URL'); + } + return url; +} + +export async function readResearchSource(target, { + fetchImpl = fetch, + lookup = dns.lookup, + timeoutMs = 10_000, + maxChars = 40_000, + signal, + redirects = 3, +} = {}) { + let current = String(target); + for (let attempt = 0; attempt <= redirects; attempt += 1) { + throwIfAborted(signal); + const url = await assertSafePublicUrl(current, { lookup }); + const timeout = AbortSignal.timeout(timeoutMs); + const combinedSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; + const response = await fetchImpl(url, { + headers: { + accept: 'text/html,text/plain,application/xhtml+xml', + 'user-agent': 'TKMind-Deep-Search/1.0', + }, + redirect: 'manual', + signal: combinedSignal, + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get('location'); + if (!location || attempt === redirects) throw new Error('Unsafe or excessive redirect'); + current = new URL(location, url).toString(); + continue; + } + if (!response.ok) throw new Error(`Reader returned ${response.status}`); + const contentType = response.headers.get('content-type') ?? ''; + if (!/text\/|application\/xhtml\+xml/i.test(contentType)) { + throw new Error(`Unsupported content type: ${contentType || 'unknown'}`); + } + const raw = (await response.text()).slice(0, maxChars * 4); + return { + url: url.toString(), + title: decodeHtml(raw.match(/]*>([\s\S]*?)<\/title>/i)?.[1] ?? '').trim().slice(0, 500), + content: (/html|xhtml/i.test(contentType) ? htmlToPlainText(raw) : raw).slice(0, maxChars), + contentType, + }; + } + throw new Error('Reader failed'); +} + +function fallbackGoals(question) { + const latinEntity = (String(question).match(/[A-Za-z][A-Za-z0-9._-]{2,}/g) ?? [])[0] ?? ''; + const candidates = []; + const add = (pattern, goal, suffix, english) => { + if (pattern.test(question)) { + candidates.push({ + goal, + query: `${question} ${suffix}`, + alternate: latinEntity ? `${latinEntity} ${english}` : `${question} ${suffix} 权威来源`, + }); + } + }; + add(/架构|组成|原理|architecture/i, '技术架构与组成', '技术架构 组件 原理', 'architecture components official documentation'); + add(/特性|功能|能力|feature|capabilit/i, '核心特性与能力', '核心特性 功能', 'features capabilities official documentation'); + add(/局限|风险|问题|争议|limitation|risk|issue/i, '主要局限、风险与争议', '局限 风险 已知问题', 'limitations known issues privacy'); + add(/市场|规模|商业|竞争|market|business/i, '市场规模与竞争格局', '市场规模 竞争格局 数据', 'market size competitors report'); + add(/对比|比较|区别|versus|compare/i, '替代方案与横向比较', '替代方案 对比', 'alternatives comparison'); + add(/趋势|未来|战略|影响|trend|future|strategy/i, '趋势、影响与未来判断', '趋势 影响 未来', 'trends roadmap future'); + const generic = [ + { goal: '核心事实与定义', query: question, alternate: latinEntity ? `${latinEntity} official documentation overview` : `${question} 权威概述` }, + { goal: '最新进展与时间线', query: `${question} 最新进展 时间线`, alternate: latinEntity ? `${latinEntity} releases changelog latest` : `${question} 最新时间线` }, + { goal: '权威及一手来源', query: `${question} 官方 报告 数据`, alternate: latinEntity ? `${latinEntity} official documentation GitHub` : `${question} 一手来源` }, + { goal: '主要参与者与方案比较', query: `${question} 主要参与者 对比`, alternate: `${question} alternatives comparison` }, + { goal: '风险、争议与反方观点', query: `${question} 风险 争议 局限`, alternate: `${question} limitations criticism` }, + { goal: '趋势、影响与未来判断', query: `${question} 趋势 影响 未来`, alternate: `${question} roadmap future` }, + ]; + const seen = new Set(candidates.map((item) => item.goal)); + return [...candidates, ...generic.filter((item) => !seen.has(item.goal))]; +} + +export function buildFallbackResearchPlan(question, depth = 'standard') { + const profile = DEPTH_PROFILES[depth] ?? DEPTH_PROFILES.standard; + return fallbackGoals(question).slice(0, profile.goals).map((item, index) => ({ + id: `goal-${index + 1}`, + goal: item.goal, + queries: [item.query, item.alternate], + sources: index === 2 ? ['web', 'paper'] : ['web'], + })); +} + +function normalizePlan(candidate, question, depth) { + const profile = DEPTH_PROFILES[depth] ?? DEPTH_PROFILES.standard; + const source = Array.isArray(candidate) ? candidate : candidate?.research_plan; + if (!Array.isArray(source) || !source.length) return buildFallbackResearchPlan(question, depth); + const plan = source.map((step, index) => ({ + id: `goal-${index + 1}`, + goal: String(step.goal ?? '').trim().slice(0, 300), + queries: (Array.isArray(step.queries) ? step.queries : step.query) + ?.map?.((query) => String(query ?? '').trim().slice(0, 500)) + .filter(Boolean) + .slice(0, 4) ?? [], + sources: (Array.isArray(step.sources) ? step.sources : ['web']) + .map((source) => String(source)) + .filter((source) => ['web', 'news', 'paper', 'code'].includes(source)) + .slice(0, 4), + })).filter((step) => step.goal && step.queries.length).slice(0, profile.goals); + return plan.length ? plan : buildFallbackResearchPlan(question, depth); +} + +export function createOpenAiCompatibleResearchLlm({ + endpoint = process.env.TKMIND_DEEP_SEARCH_LLM_URL, + apiKey = process.env.TKMIND_DEEP_SEARCH_LLM_API_KEY, + model = process.env.TKMIND_DEEP_SEARCH_LLM_MODEL, + fetchImpl = fetch, + timeoutMs = 90_000, +} = {}) { + if (!endpoint || !model) return null; + async function complete(messages, { json = false } = {}) { + const response = await fetchImpl(endpoint, { + method: 'POST', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + 'content-type': 'application/json', + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + messages, + temperature: 0.2, + ...(json ? { response_format: { type: 'json_object' } } : {}), + }), + }); + if (!response.ok) throw new Error(`Research LLM returned ${response.status}`); + const body = await response.json(); + return String(body.choices?.[0]?.message?.content ?? '').trim(); + } + return { + async plan(question, depth) { + const content = await complete([ + { + role: 'system', + content: 'You are a research planner. Return JSON {"research_plan":[{"goal":"","queries":[""],"sources":["web"]}]}. Create distinct, verifiable goals and search queries.', + }, + { role: 'user', content: `Depth: ${depth}\nQuestion: ${question}` }, + ], { json: true }); + return JSON.parse(content); + }, + async synthesize({ question, plan, evidence }) { + return complete([ + { + role: 'system', + content: 'Write a rigorous Markdown research report using only the supplied evidence. Cite every factual claim with [n]. Include executive summary, findings by research goal, uncertainties, and sources. Never invent citations.', + }, + { + role: 'user', + content: JSON.stringify({ question, plan, evidence }, null, 2).slice(0, 120_000), + }, + ]); + }, + }; +} + +export function createPortalGatewayResearchLlmResolver({ + endpoint = process.env.TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL + || 'http://127.0.0.1:8081/api/internal/deep-search/llm', + secret = process.env.TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET + || process.env.TKMIND_DEEP_SEARCH_SECRET, + fetchImpl = fetch, + timeoutMs = 90_000, +} = {}) { + const resolver = async ({ providerKeyId, model }) => { + if (!providerKeyId || !model) throw new Error('Deep Search LLM provider and model are required'); + if (!secret) throw new Error('Deep Search LLM gateway secret is not configured'); + async function complete(messages, { json = false } = {}) { + const response = await fetchImpl(endpoint, { + method: 'POST', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + authorization: `Bearer ${secret}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + providerKeyId, + model, + messages, + temperature: 0.2, + json, + }), + }); + const body = await response.json().catch(() => ({})); + if (!response.ok || !body.ok || !body.reply) { + throw new Error(body.message || `Deep Search LLM gateway returned ${response.status}`); + } + return String(body.reply).trim(); + } + return { + async plan(question, depth) { + const content = await complete([ + { + role: 'system', + content: 'You are a research planner. Return JSON {"research_plan":[{"goal":"","queries":[""],"sources":["web"]}]}. Create distinct, verifiable goals and search queries.', + }, + { role: 'user', content: `Depth: ${depth}\nQuestion: ${question}` }, + ], { json: true }); + return JSON.parse(content); + }, + async synthesize({ question, plan, evidence }) { + return complete([ + { + role: 'system', + content: 'Write a rigorous Markdown research report using only the supplied evidence. Cite every factual claim with [n]. Include executive summary, findings by research goal, uncertainties, and sources. Never invent citations.', + }, + { + role: 'user', + content: JSON.stringify({ question, plan, evidence }, null, 2).slice(0, 120_000), + }, + ]); + }, + }; + }; + resolver.configured = Boolean(endpoint && secret); + return resolver; +} + +function sourceAuthority(url) { + try { + const hostname = new URL(url).hostname; + if (/\.(gov|edu)$/.test(hostname) || hostname === 'arxiv.org') return 0.2; + if (hostname === 'github.com' || hostname.endsWith('.org')) return 0.12; + } catch { + return 0; + } + return 0; +} + +function scoreSource(source, question, goal = '') { + const haystack = `${source.title} ${source.snippet} ${source.content}`; + const base = clamp( + lexicalScore(question, haystack) * 0.45 + + lexicalScore(goal, haystack) * 0.35 + + sourceAuthority(source.url) + + (/official|documentation|docs|官方|文档/i.test(source.title) ? 0.08 : 0) + + (source.content?.length > 1000 ? 0.05 : 0) + + (source.rank ? 0.05 / source.rank : 0), + 0, + 1, + ); + const entity = (String(question).match(/[A-Za-z][A-Za-z0-9._-]{2,}/g) ?? [])[0]; + return entity && !haystack.toLowerCase().includes(entity.toLowerCase()) ? base * 0.25 : base; +} + +function selectSourcesForReading(sourceMap, plan, maxSources) { + const all = [...sourceMap.values()].sort((a, b) => b.score - a.score); + const selected = []; + const seen = new Set(); + for (const step of plan) { + for (const source of all.filter((item) => item.goal === step.goal).slice(0, 2)) { + if (seen.has(source.url)) continue; + seen.add(source.url); + selected.push(source); + } + } + for (const source of all) { + if (selected.length >= maxSources) break; + if (seen.has(source.url)) continue; + seen.add(source.url); + selected.push(source); + } + return selected.slice(0, maxSources); +} + +function splitSentences(content) { + return String(content ?? '') + .split(/(?<=[。！？.!?])\s+|\n+/u) + .map((sentence) => sentence.trim()) + .filter((sentence) => sentence.length >= 35 && sentence.length <= 1200); +} + +export function extractEvidence(source, { question, goal }) { + const normalizedTitle = String(source.title ?? '').toLowerCase().replace(/\s+/g, ' ').trim(); + const sentences = splitSentences(source.content || source.snippet) + .filter((claim) => { + const normalized = claim.toLowerCase().replace(/\s+/g, ' ').trim(); + if (!normalized || normalized === normalizedTitle) return false; + if (normalized.startsWith(normalizedTitle) && normalized.length < normalizedTitle.length + 30) return false; + return !/contents\s+menu\s+expand|light mode\s+dark mode|sign in\s+sign up|data-hydro-click|class="|aria-[a-z-]+=/i.test(normalized); + }); + const ranked = sentences + .map((claim) => ({ + claim, + score: lexicalScore(`${question} ${goal}`, claim) + Math.min(0.25, claim.length / 1000), + })) + .sort((a, b) => b.score - a.score) + .slice(0, 2); + if (!ranked.length && source.snippet) ranked.push({ claim: source.snippet, score: 0.2 }); + return ranked.map((item) => ({ + claim: item.claim, + confidence: clamp(0.45 + item.score * 0.45 + sourceAuthority(source.url), 0.45, 0.95), + url: source.url, + title: source.title, + goal, + })); +} + +async function runBounded(items, limit, worker) { + const output = new Array(items.length); + let cursor = 0; + async function run() { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + output[index] = await worker(items[index], index); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run)); + return output; +} + +function reportFromEvidence({ question, plan, evidence, sources }) { + const sourceIndex = new Map(sources.map((source, index) => [source.url, index + 1])); + const lines = [ + `# ${question}`, + '', + '## 执行摘要', + '', + ]; + const strongest = [...evidence].sort((a, b) => b.confidence - a.confidence).slice(0, 5); + if (!strongest.length) { + lines.push('当前检索未获得足够的可验证证据，建议调整问题或检查搜索服务。'); + } else { + for (const item of strongest) lines.push(`- ${item.claim} [${sourceIndex.get(item.url)}]`); + } + for (const step of plan) { + lines.push('', `## ${step.goal}`, ''); + const items = evidence.filter((item) => item.goal === step.goal).slice(0, 5); + if (!items.length) lines.push('该研究目标暂未获得足够证据。'); + for (const item of items) { + lines.push(`- ${item.claim} [${sourceIndex.get(item.url)}]`); + } + } + lines.push('', '## 不确定性与局限', ''); + lines.push('- 报告仅基于本次检索可访问的公开网页；付费墙、动态页面和非文本资料可能未被读取。'); + lines.push('- 自动提取的事实仍建议对照一手来源复核，尤其是数字、日期和预测。'); + lines.push('', '## 来源', ''); + sources.forEach((source, index) => { + lines.push(`${index + 1}. [${source.title || source.url}](${source.url}) — ${source.provider}`); + }); + return lines.join('\n'); +} + +function citationNumbers(markdown) { + return [...String(markdown).matchAll(/\[(\d+)\]/g)].map((match) => Number(match[1])); +} + +function validLlmReport(markdown, sourceCount) { + const citations = citationNumbers(markdown); + return markdown.length >= 200 + && citations.length > 0 + && citations.every((number) => number >= 1 && number <= sourceCount); +} + +export function createSearxngResearchProvider({ + endpoint = process.env.TKMIND_DEEP_SEARCH_SEARXNG_URL || process.env.TKMIND_SEARCH_SEARXNG_URL || 'http://127.0.0.1:8080/search', + fetchImpl = fetch, +} = {}) { + return { + name: 'searxng', + async search(query, { limit, signal } = {}) { + throwIfAborted(signal); + return searchSearxng(query, { limit, endpoint, fetchImpl, signal }); + }, + }; +} + +export function createDeepSearchEngine({ + store, + searchProvider = createSearxngResearchProvider(), + reader = readResearchSource, + llm = createOpenAiCompatibleResearchLlm(), + llmResolver = null, + memorySink = null, + idFactory = () => randomUUID(), + now = () => Date.now(), + concurrency = 3, +} = {}) { + if (!store) throw new Error('Deep Search store is required'); + const running = new Map(); + const emitter = new EventEmitter(); + + function publish(taskId, type, payload = {}) { + store.appendEvent(taskId, type, payload); + emitter.emit(taskId, { type, payload, createdAt: now() }); + } + + async function planResearch(question, depth, activeLlm) { + if (activeLlm?.plan) { + try { + return normalizePlan(await activeLlm.plan(question, depth), question, depth); + } catch { + // The deterministic planner keeps Deep Search available when the LLM is unavailable. + } + } + return buildFallbackResearchPlan(question, depth); + } + + async function executeTask(taskId, signal) { + const task = store.getTask(taskId, { includeSources: false, includeEvents: false }); + const profile = DEPTH_PROFILES[task.depth] ?? DEPTH_PROFILES.standard; + try { + store.updateTask(taskId, { status: 'researching', phase: 'planning', progress: 5 }); + publish(taskId, 'phase', { phase: 'planning', progress: 5 }); + let activeLlm = llm; + if (task.llmProviderKeyId && typeof llmResolver === 'function') { + try { + activeLlm = await llmResolver({ + providerKeyId: task.llmProviderKeyId, + model: task.llmModel, + }); + publish(taskId, 'llm_selected', { + providerKeyId: task.llmProviderKeyId, + model: task.llmModel, + }); + } catch (error) { + activeLlm = null; + publish(taskId, 'llm_error', { message: String(error?.message ?? error) }); + } + } + const plan = await planResearch(task.question, task.depth, activeLlm); + throwIfAborted(signal); + store.updateTask(taskId, { plan, phase: 'searching', progress: 12 }); + publish(taskId, 'plan', { plan }); + + const sourcesByUrl = new Map(); + for (let round = 0; round < profile.rounds; round += 1) { + throwIfAborted(signal); + const jobs = plan.map((step) => ({ + step, + query: step.queries[round % step.queries.length], + round, + })); + const batches = await runBounded(jobs, concurrency, async ({ step, query }) => { + try { + const results = await searchProvider.search(query, { + limit: profile.resultsPerQuery, + source: step.sources[0], + signal, + }); + publish(taskId, 'search', { goal: step.goal, query, count: results.length, round: round + 1 }); + return results.map((result) => ({ ...result, goal: step.goal, query })); + } catch (error) { + if (error?.name === 'AbortError') throw error; + publish(taskId, 'search_error', { goal: step.goal, query, message: error.message }); + return []; + } + }); + for (const source of batches.flat()) { + const url = canonicalizeUrl(source.url); + if (!url) continue; + const candidate = { + ...source, + url, + provider: source.provider || source.source || searchProvider.name || 'unknown', + score: scoreSource(source, task.question, source.goal), + }; + const previous = sourcesByUrl.get(url); + if (!previous || candidate.score > previous.score) sourcesByUrl.set(url, candidate); + } + const progress = 12 + Math.round(((round + 1) / profile.rounds) * 28); + store.updateTask(taskId, { progress, phase: 'searching' }); + publish(taskId, 'progress', { phase: 'searching', progress, sources: sourcesByUrl.size }); + } + + const ranked = selectSourcesForReading(sourcesByUrl, plan, profile.maxSources); + store.updateTask(taskId, { phase: 'reading', progress: 45 }); + publish(taskId, 'phase', { phase: 'reading', progress: 45, sources: ranked.length }); + const readSources = await runBounded(ranked, concurrency, async (source, index) => { + throwIfAborted(signal); + let enriched = source; + try { + const document = await reader(source.url, { signal }); + enriched = { + ...source, + title: document.title || source.title, + content: document.content, + }; + } catch (error) { + if (error?.name === 'AbortError') throw error; + publish(taskId, 'read_error', { url: source.url, message: error.message }); + } + enriched.score = scoreSource(enriched, task.question, enriched.goal); + store.addSource(taskId, enriched); + const progress = 45 + Math.round(((index + 1) / Math.max(1, ranked.length)) * 25); + publish(taskId, 'read', { url: source.url, progress }); + return enriched; + }); + + throwIfAborted(signal); + const finalSources = readSources.sort((a, b) => b.score - a.score); + const evidence = []; + const seenClaims = new Set(); + for (const source of finalSources) { + for (const item of extractEvidence(source, { question: task.question, goal: source.goal })) { + const key = item.claim.toLowerCase().replace(/\W+/gu, '').slice(0, 180); + if (!key || seenClaims.has(key)) continue; + seenClaims.add(key); + evidence.push(item); + } + } + store.updateTask(taskId, { phase: 'synthesizing', progress: 78 }); + publish(taskId, 'phase', { phase: 'synthesizing', progress: 78, evidence: evidence.length }); + let report = ''; + if (activeLlm?.synthesize && finalSources.length) { + try { + const numberedEvidence = evidence.map((item) => ({ + ...item, + citation: finalSources.findIndex((source) => source.url === item.url) + 1, + })); + const candidate = await activeLlm.synthesize({ + question: task.question, + plan, + evidence: numberedEvidence, + }); + if (validLlmReport(candidate, finalSources.length)) report = candidate; + } catch { + // Fall through to the citation-safe deterministic report. + } + } + if (!report) report = reportFromEvidence({ question: task.question, plan, evidence, sources: finalSources }); + throwIfAborted(signal); + const completedAt = now(); + store.updateTask(taskId, { + status: 'completed', + phase: 'completed', + progress: 100, + report, + completedAt, + }); + store.saveMemory({ + id: `research:${taskId}`, + userId: task.userId, + topic: task.question, + summary: report.slice(0, 4000), + sourceTaskId: taskId, + }); + if (typeof memorySink === 'function' && task.userId) { + try { + await memorySink({ + userId: task.userId, + taskId, + topic: task.question, + summary: report.slice(0, 4000), + sources: finalSources.map((source) => ({ + title: source.title, + url: source.url, + provider: source.provider, + })), + }); + publish(taskId, 'memory_saved', { userId: task.userId }); + } catch (error) { + publish(taskId, 'memory_error', { message: String(error?.message ?? error) }); + } + } + publish(taskId, 'completed', { + progress: 100, + sources: finalSources.length, + evidence: evidence.length, + }); + } catch (error) { + if (error?.name === 'AbortError' || signal.aborted) { + store.updateTask(taskId, { + status: 'cancelled', + phase: 'cancelled', + error: null, + completedAt: now(), + }); + publish(taskId, 'cancelled', {}); + } else { + store.updateTask(taskId, { + status: 'failed', + phase: 'failed', + error: String(error?.message ?? error).slice(0, 2000), + completedAt: now(), + }); + publish(taskId, 'failed', { message: String(error?.message ?? error) }); + } + } finally { + running.delete(taskId); + } + } + + return { + start({ + question, + depth = 'standard', + userId = null, + llmProviderKeyId = '', + llmModel = '', + } = {}) { + const normalizedQuestion = String(question ?? '').trim(); + if (!normalizedQuestion || normalizedQuestion.length > 2000) { + throw new Error('question must be 1-2000 characters'); + } + const normalizedDepth = Object.hasOwn(DEPTH_PROFILES, depth) ? depth : 'standard'; + const normalizedProviderKeyId = /^[a-zA-Z0-9._:-]{1,128}$/.test(String(llmProviderKeyId)) + ? String(llmProviderKeyId) + : ''; + const normalizedModel = normalizedProviderKeyId + ? String(llmModel ?? '').trim().slice(0, 200) + : ''; + const taskId = idFactory(); + store.createTask({ + id: taskId, + userId, + question: normalizedQuestion, + depth: normalizedDepth, + llmProviderKeyId: normalizedProviderKeyId, + llmModel: normalizedModel, + }); + const controller = new AbortController(); + running.set(taskId, controller); + queueMicrotask(() => executeTask(taskId, controller.signal)); + return { taskId, status: 'queued', service: 'tkmind-deep-search' }; + }, + + async search({ query, limit = 10, type = 'web' } = {}) { + const normalizedQuery = String(query ?? '').trim(); + if (!normalizedQuery || normalizedQuery.length > 500) throw new Error('query must be 1-500 characters'); + const results = await searchProvider.search(normalizedQuery, { + limit: clamp(Number(limit) || 10, 1, 20), + source: type, + }); + return results; + }, + + getTask(id) { + return store.getTask(id); + }, + + listTasks(limit) { + return store.listTasks(limit); + }, + + cancel(id) { + const task = store.getTask(id, { includeSources: false, includeEvents: false }); + if (!task) return null; + if (['completed', 'failed', 'cancelled'].includes(task.status)) return task; + running.get(String(id))?.abort(); + store.updateTask(id, { status: 'cancelling', phase: 'cancelling' }); + publish(id, 'cancelling', {}); + return store.getTask(id); + }, + + subscribe(id, listener) { + emitter.on(String(id), listener); + return () => emitter.off(String(id), listener); + }, + + getHealth() { + return { + ok: true, + service: 'tkmind-deep-search', + running: running.size, + tasks: store.getStats(), + llmEnabled: Boolean(llm || llmResolver?.configured), + llmMode: llmResolver?.configured ? 'portal-gateway' : (llm ? 'direct' : 'deterministic'), + provider: searchProvider.name ?? 'custom', + }; + }, + }; +} diff --git a/deep-search-llm-gateway.mjs b/deep-search-llm-gateway.mjs new file mode 100644 index 0000000..27f8245 --- /dev/null +++ b/deep-search-llm-gateway.mjs @@ -0,0 +1,63 @@ +import crypto from 'node:crypto'; + +function secureEqual(left, right) { + const a = Buffer.from(String(left ?? '')); + const b = Buffer.from(String(right ?? '')); + return a.length === b.length && crypto.timingSafeEqual(a, b); +} + +function normalizeMessages(value) { + if (!Array.isArray(value) || !value.length || value.length > 20) return null; + const messages = value.map((message) => ({ + role: ['system', 'user', 'assistant'].includes(message?.role) ? message.role : '', + content: String(message?.content ?? '').trim(), + })); + if (messages.some((message) => !message.role || !message.content || message.content.length > 120_000)) { + return null; + } + if (messages.reduce((total, message) => total + message.content.length, 0) > 150_000) return null; + return messages; +} + +export async function executeDeepSearchLlmGateway({ + llmProviderService, + expectedSecret, + providedSecret, + input = {}, +} = {}) { + if (!expectedSecret || !secureEqual(expectedSecret, providedSecret)) { + return { status: 401, body: { ok: false, message: 'Deep Search LLM gateway credential is invalid' } }; + } + if (!llmProviderService?.createChatCompletion) { + return { status: 503, body: { ok: false, message: 'LLM provider service is unavailable' } }; + } + const providerKeyId = String(input.providerKeyId ?? '').trim(); + const model = String(input.model ?? '').trim(); + const messages = normalizeMessages(input.messages); + if (!/^[a-zA-Z0-9._:-]{1,128}$/.test(providerKeyId) || !model || model.length > 200 || !messages) { + return { status: 400, body: { ok: false, message: 'Invalid Deep Search LLM request' } }; + } + const result = await llmProviderService.createChatCompletion({ + providerKeyId, + model, + messages, + temperature: Math.max(0, Math.min(1, Number(input.temperature ?? 0.2) || 0.2)), + }); + if (!result?.ok) { + return { + status: Number(result?.status) >= 400 ? Number(result.status) : 502, + body: { ok: false, message: result?.message || 'Deep Search LLM request failed' }, + }; + } + return { + status: 200, + body: { + ok: true, + reply: result.reply, + providerKeyId: result.providerKeyId, + providerId: result.providerId, + model: result.model, + usage: result.usage ?? null, + }, + }; +} diff --git a/deep-search-server.mjs b/deep-search-server.mjs new file mode 100644 index 0000000..b0004fa --- /dev/null +++ b/deep-search-server.mjs @@ -0,0 +1,228 @@ +import http from 'node:http'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { + createDeepSearchEngine, + createPortalGatewayResearchLlmResolver, +} from './deep-search-engine.mjs'; +import { createDeepSearchStore } from './deep-search-store.mjs'; +import { searchGithubCode } from './mindsearch-providers.mjs'; + +function sendJson(res, status, body) { + const payload = JSON.stringify(body); + res.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(payload), + 'cache-control': 'no-store', + }); + res.end(payload); +} + +async function readJson(req, { maxBytes = 1_000_000 } = {}) { + const chunks = []; + let size = 0; + for await (const chunk of req) { + size += chunk.length; + if (size > maxBytes) { + const error = new Error('Request body is too large'); + error.status = 413; + throw error; + } + chunks.push(chunk); + } + if (!chunks.length) return {}; + try { + return JSON.parse(Buffer.concat(chunks).toString('utf8')); + } catch { + const error = new Error('Request body must be valid JSON'); + error.status = 400; + throw error; + } +} + +function publicTask(task, { includeContent = false, includeEvents = false } = {}) { + if (!task) return null; + return { + ...task, + sources: task.sources?.map((source) => ({ + ...source, + ...(!includeContent ? { content: undefined } : {}), + })), + ...(!includeEvents ? { events: undefined } : {}), + }; +} + +function matchTaskPath(pathname, suffix = '') { + const match = pathname.match(new RegExp(`^/v1/research/([^/]+)${suffix}$`)); + return match ? decodeURIComponent(match[1]) : null; +} + +function canAccessTask(task, requestUserId) { + return !requestUserId || !task?.userId || task.userId === requestUserId; +} + +export function createDeepSearchHttpServer({ + engine, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET || '', + githubToken = process.env.GITHUB_TOKEN || '', + githubSearch = searchGithubCode, + logger = console, +} = {}) { + if (!engine) throw new Error('Deep Search engine is required'); + return http.createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://deep-search.local'); + if (url.pathname === '/health' && req.method === 'GET') { + return sendJson(res, 200, { + ...engine.getHealth(), + providers: { + github: { configured: Boolean(githubToken) }, + }, + }); + } + if (secret && req.headers['x-secret-key'] !== secret) { + return sendJson(res, 401, { code: 'UNAUTHORIZED', message: 'Invalid service secret' }); + } + const requestUserId = String(req.headers['x-user-id'] ?? '').trim(); + try { + if (url.pathname === '/v1/providers/github/search' && req.method === 'POST') { + if (!githubToken) { + const error = new Error('GitHub search token is not configured'); + error.status = 503; + throw error; + } + const input = await readJson(req, { maxBytes: 32_000 }); + const query = String(input.query ?? '').trim(); + if (!query || query.length > 500) { + const error = new Error('query must be 1-500 characters'); + error.status = 400; + throw error; + } + const limit = Math.max(1, Math.min(20, Number(input.limit ?? 10) || 10)); + const results = await githubSearch(query, { + limit, + token: githubToken, + }); + return sendJson(res, 200, { + query, + results, + service: 'tkmind-deep-search', + }); + } + if (url.pathname === '/v1/search' && req.method === 'POST') { + const input = await readJson(req); + const results = await engine.search(input); + return sendJson(res, 200, { query: input.query, results, service: 'tkmind-deep-search' }); + } + if (url.pathname === '/v1/research' && req.method === 'POST') { + const input = await readJson(req); + const task = engine.start({ + ...input, + userId: requestUserId || input.userId || null, + llmProviderKeyId: input.llmProviderKeyId || input.llm_provider_key_id || '', + llmModel: input.llmModel || input.llm_model || '', + }); + return sendJson(res, 202, { + task_id: task.taskId, + status: task.status, + service: task.service, + }); + } + if (url.pathname === '/v1/research' && req.method === 'GET') { + const tasks = engine.listTasks(url.searchParams.get('limit')) + .filter((task) => canAccessTask(task, requestUserId)); + return sendJson(res, 200, { tasks }); + } + const eventTaskId = matchTaskPath(url.pathname, '/events'); + if (eventTaskId && req.method === 'GET') { + const task = engine.getTask(eventTaskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + if (!canAccessTask(task, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'cache-control': 'no-cache, no-transform', + connection: 'keep-alive', + }); + for (const event of task.events ?? []) { + res.write(`id: ${event.seq}\nevent: ${event.type}\ndata: ${JSON.stringify(event.payload)}\n\n`); + } + const unsubscribe = engine.subscribe(eventTaskId, (event) => { + res.write(`event: ${event.type}\ndata: ${JSON.stringify(event.payload)}\n\n`); + if (['completed', 'failed', 'cancelled'].includes(event.type)) res.end(); + }); + const heartbeat = setInterval(() => res.write(': heartbeat\n\n'), 15_000); + const cleanup = () => { + clearInterval(heartbeat); + unsubscribe(); + }; + req.once('close', cleanup); + res.once('close', cleanup); + return; + } + const taskId = matchTaskPath(url.pathname); + if (taskId && req.method === 'GET') { + const task = engine.getTask(taskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + if (!canAccessTask(task, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + return sendJson(res, 200, publicTask(task, { + includeContent: url.searchParams.get('include_content') === '1', + includeEvents: url.searchParams.get('include_events') === '1', + })); + } + if (taskId && req.method === 'DELETE') { + const existing = engine.getTask(taskId); + if (existing && !canAccessTask(existing, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + const task = engine.cancel(taskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + return sendJson(res, 202, publicTask(task)); + } + return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Route not found' }); + } catch (error) { + logger.warn?.('[deep-search] request failed', error); + return sendJson(res, Number(error?.status) || 400, { + code: 'REQUEST_FAILED', + message: String(error?.message ?? error), + }); + } + }); +} + +export function startDeepSearchServer({ + host = process.env.TKMIND_DEEP_SEARCH_HOST || '127.0.0.1', + port = Number(process.env.TKMIND_DEEP_SEARCH_PORT || 20100), + databasePath = process.env.TKMIND_DEEP_SEARCH_DB + || path.join(process.cwd(), '.deep-search', 'research.sqlite'), + logger = console, +} = {}) { + const store = createDeepSearchStore({ databasePath }); + const engine = createDeepSearchEngine({ + store, + llm: null, + llmResolver: createPortalGatewayResearchLlmResolver(), + }); + const server = createDeepSearchHttpServer({ engine, logger }); + server.listen(port, host, () => { + logger.info?.(`[deep-search] listening on http://${host}:${port}`); + }); + const close = async () => { + await new Promise((resolve) => server.close(resolve)); + store.close(); + }; + return { server, engine, store, close }; +} + +const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ''; +if (import.meta.url === invokedPath) { + const runtime = startDeepSearchServer(); + const shutdown = async () => { + await runtime.close(); + process.exit(0); + }; + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); +} diff --git a/deep-search-store.mjs b/deep-search-store.mjs new file mode 100644 index 0000000..0ca7b87 --- /dev/null +++ b/deep-search-store.mjs @@ -0,0 +1,276 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +const TASK_COLUMNS = new Set([ + 'status', + 'progress', + 'phase', + 'plan_json', + 'report', + 'error', + 'updated_at', + 'completed_at', +]); + +function parseJson(value, fallback) { + if (!value) return fallback; + try { + return JSON.parse(value); + } catch { + return fallback; + } +} + +function mapTask(row) { + if (!row) return null; + return { + id: row.id, + userId: row.user_id || null, + question: row.question, + depth: row.depth, + llmProviderKeyId: row.llm_provider_key_id || '', + llmModel: row.llm_model || '', + status: row.status, + progress: row.progress, + phase: row.phase, + plan: parseJson(row.plan_json, []), + report: row.report || '', + error: row.error || null, + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at || null, + }; +} + +function mapSource(row) { + return { + id: row.id, + taskId: row.task_id, + url: row.url, + title: row.title, + snippet: row.snippet, + content: row.content, + provider: row.provider, + score: row.score, + query: row.query, + createdAt: row.created_at, + }; +} + +export function createDeepSearchStore({ + databasePath = process.env.TKMIND_DEEP_SEARCH_DB || path.join(process.cwd(), '.deep-search', 'research.sqlite'), + now = () => Date.now(), +} = {}) { + if (databasePath !== ':memory:') { + fs.mkdirSync(path.dirname(path.resolve(databasePath)), { recursive: true }); + } + const db = new DatabaseSync(databasePath); + db.exec(` + PRAGMA journal_mode = WAL; + PRAGMA foreign_keys = ON; + CREATE TABLE IF NOT EXISTS research_tasks ( + id TEXT PRIMARY KEY, + user_id TEXT, + question TEXT NOT NULL, + depth TEXT NOT NULL, + llm_provider_key_id TEXT NOT NULL DEFAULT '', + llm_model TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL, + progress INTEGER NOT NULL DEFAULT 0, + phase TEXT NOT NULL DEFAULT 'queued', + plan_json TEXT NOT NULL DEFAULT '[]', + report TEXT NOT NULL DEFAULT '', + error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ); + CREATE TABLE IF NOT EXISTS research_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + url TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + snippet TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT 'unknown', + score REAL NOT NULL DEFAULT 0, + query TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + UNIQUE(task_id, url) + ); + CREATE TABLE IF NOT EXISTS research_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + type TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS research_memories ( + id TEXT PRIMARY KEY, + user_id TEXT, + topic TEXT NOT NULL, + summary TEXT NOT NULL, + source_task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_research_tasks_status ON research_tasks(status, updated_at DESC); + CREATE INDEX IF NOT EXISTS idx_research_sources_task ON research_sources(task_id, score DESC); + CREATE INDEX IF NOT EXISTS idx_research_events_task ON research_events(task_id, seq); + `); + const taskColumns = new Set(db.prepare('PRAGMA table_info(research_tasks)').all().map((row) => row.name)); + if (!taskColumns.has('llm_provider_key_id')) { + db.exec("ALTER TABLE research_tasks ADD COLUMN llm_provider_key_id TEXT NOT NULL DEFAULT ''"); + } + if (!taskColumns.has('llm_model')) { + db.exec("ALTER TABLE research_tasks ADD COLUMN llm_model TEXT NOT NULL DEFAULT ''"); + } + + const createTaskStmt = db.prepare(` + INSERT INTO research_tasks + (id, user_id, question, depth, llm_provider_key_id, llm_model, status, progress, phase, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'queued', 0, 'queued', ?, ?) + `); + const getTaskStmt = db.prepare('SELECT * FROM research_tasks WHERE id = ?'); + const listTasksStmt = db.prepare('SELECT * FROM research_tasks ORDER BY created_at DESC LIMIT ?'); + const sourcesStmt = db.prepare('SELECT * FROM research_sources WHERE task_id = ? ORDER BY score DESC, id ASC'); + const eventsStmt = db.prepare('SELECT * FROM research_events WHERE task_id = ? ORDER BY seq ASC'); + const addSourceStmt = db.prepare(` + INSERT INTO research_sources + (task_id, url, title, snippet, content, provider, score, query, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(task_id, url) DO UPDATE SET + title = excluded.title, + snippet = excluded.snippet, + content = CASE + WHEN length(excluded.content) > length(research_sources.content) THEN excluded.content + ELSE research_sources.content + END, + provider = excluded.provider, + score = MAX(research_sources.score, excluded.score), + query = excluded.query + `); + const appendEventStmt = db.prepare(` + INSERT INTO research_events (task_id, type, payload_json, created_at) + VALUES (?, ?, ?, ?) + `); + const saveMemoryStmt = db.prepare(` + INSERT INTO research_memories (id, user_id, topic, summary, source_task_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET summary = excluded.summary + `); + + function getTask(id, { includeSources = true, includeEvents = true } = {}) { + const task = mapTask(getTaskStmt.get(String(id))); + if (!task) return null; + if (includeSources) task.sources = sourcesStmt.all(task.id).map(mapSource); + if (includeEvents) { + task.events = eventsStmt.all(task.id).map((event) => ({ + seq: event.seq, + type: event.type, + payload: parseJson(event.payload_json, {}), + createdAt: event.created_at, + })); + } + return task; + } + + return { + createTask({ + id, + userId = null, + question, + depth, + llmProviderKeyId = '', + llmModel = '', + }) { + const timestamp = now(); + createTaskStmt.run( + id, + userId, + question, + depth, + llmProviderKeyId, + llmModel, + timestamp, + timestamp, + ); + appendEventStmt.run(id, 'queued', JSON.stringify({ depth }), timestamp); + return getTask(id); + }, + + updateTask(id, patch = {}) { + const normalized = { ...patch, updated_at: patch.updatedAt ?? now() }; + if (Array.isArray(normalized.plan)) { + normalized.plan_json = JSON.stringify(normalized.plan); + delete normalized.plan; + } + if (Object.hasOwn(normalized, 'completedAt')) { + normalized.completed_at = normalized.completedAt; + delete normalized.completedAt; + } + if (Object.hasOwn(normalized, 'updatedAt')) delete normalized.updatedAt; + const entries = Object.entries(normalized).filter(([key]) => TASK_COLUMNS.has(key)); + if (entries.length) { + const assignments = entries.map(([key]) => `${key} = ?`).join(', '); + db.prepare(`UPDATE research_tasks SET ${assignments} WHERE id = ?`) + .run(...entries.map(([, value]) => value), String(id)); + } + return getTask(id); + }, + + addSource(taskId, source = {}) { + addSourceStmt.run( + String(taskId), + String(source.url ?? ''), + String(source.title ?? '').slice(0, 1000), + String(source.snippet ?? '').slice(0, 4000), + String(source.content ?? '').slice(0, 100_000), + String(source.provider ?? 'unknown').slice(0, 100), + Number(source.score ?? 0) || 0, + String(source.query ?? '').slice(0, 1000), + now(), + ); + }, + + appendEvent(taskId, type, payload = {}) { + appendEventStmt.run(String(taskId), String(type), JSON.stringify(payload), now()); + }, + + saveMemory({ id, userId = null, topic, summary, sourceTaskId }) { + saveMemoryStmt.run(id, userId, topic, summary, sourceTaskId, now()); + }, + + getTask, + + listTasks(limit = 50) { + const bounded = Math.max(1, Math.min(200, Number(limit) || 50)); + return listTasksStmt.all(bounded).map(mapTask); + }, + + listMemories({ userId = null, limit = 50 } = {}) { + const bounded = Math.max(1, Math.min(200, Number(limit) || 50)); + const rows = userId + ? db.prepare('SELECT * FROM research_memories WHERE user_id = ? ORDER BY created_at DESC LIMIT ?') + .all(String(userId), bounded) + : db.prepare('SELECT * FROM research_memories ORDER BY created_at DESC LIMIT ?').all(bounded); + return rows.map((row) => ({ + id: row.id, + userId: row.user_id || null, + topic: row.topic, + summary: row.summary, + sourceTaskId: row.source_task_id, + createdAt: row.created_at, + })); + }, + + getStats() { + const rows = db.prepare('SELECT status, COUNT(*) AS count FROM research_tasks GROUP BY status').all(); + return Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])); + }, + + close() { + db.close(); + }, + }; +} diff --git a/deep-search.test.mjs b/deep-search.test.mjs new file mode 100644 index 0000000..eadb154 --- /dev/null +++ b/deep-search.test.mjs @@ -0,0 +1,352 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + assertSafePublicUrl, + buildFallbackResearchPlan, + canonicalizeUrl, + createDeepSearchEngine, + createPortalGatewayResearchLlmResolver, + extractEvidence, + htmlToPlainText, + readResearchSource, +} from './deep-search-engine.mjs'; +import { executeDeepSearchLlmGateway } from './deep-search-llm-gateway.mjs'; +import { createDeepSearchHttpServer } from './deep-search-server.mjs'; +import { createDeepSearchStore } from './deep-search-store.mjs'; + +async function waitFor(check, { timeoutMs = 3000 } = {}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const value = await check(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('Timed out waiting for condition'); +} + +function createMockEngine({ id = 'task-complete' } = {}) { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + const searches = []; + const engine = createDeepSearchEngine({ + store, + idFactory: () => id, + searchProvider: { + name: 'mock-search', + async search(query, { limit }) { + searches.push(query); + const slug = encodeURIComponent(query).slice(0, 60); + return Array.from({ length: Math.min(2, limit) }, (_, index) => ({ + title: `${query} source ${index + 1}`, + url: `https://example.com/${slug}/${index + 1}?utm_source=test`, + snippet: `Evidence about ${query}, with verifiable details and an independent source.`, + source: 'mock', + rank: index + 1, + })); + }, + }, + reader: async (url) => ({ + url, + title: `Document ${url}`, + content: 'This document provides verifiable evidence about the research question and explains the relevant facts in sufficient detail. It also identifies limitations and cites a primary source.', + contentType: 'text/html', + }), + }); + return { engine, store, searches }; +} + +test('Deep Search planner scales goals by depth and canonicalizes evidence URLs', () => { + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'quick').length, 3); + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'standard').length, 4); + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'deep').length, 6); + assert.equal( + canonicalizeUrl('https://Example.com/report?utm_source=x&id=3#details'), + 'https://example.com/report?id=3', + ); + assert.match(htmlToPlainText('A

Hello & world

'), /Hello & world/); +}); + +test('Deep Search reader blocks private destinations and extracts public HTML', async () => { + await assert.rejects( + () => assertSafePublicUrl('http://127.0.0.1/private'), + /Unsafe source URL/, + ); + const document = await readResearchSource('https://example.com/article', { + lookup: async () => [{ address: '93.184.216.34', family: 4 }], + fetchImpl: async () => new Response( + 'Research
Useful evidence for the report.
', + { status: 200, headers: { 'content-type': 'text/html' } }, + ), + }); + assert.equal(document.title, 'Research'); + assert.match(document.content, /Useful evidence/); + assert.doesNotMatch(document.content, /ignore/); +}); + +test('Deep Search evidence extraction drops page chrome and HTML attribute noise', () => { + const evidence = extractEvidence({ + title: 'SearXNG privacy', + url: 'https://example.com/privacy', + content: [ + 'data-hydro-click="{\\"event_type\\":\\"authentication.click\\"}" class="HeaderMenu-link" this is navigation noise that must not become evidence.', + 'SearXNG protects search privacy by avoiding user profiling and by proxying requests to multiple upstream search engines without forwarding browser cookies.', + ].join('\n'), + }, { + question: 'How does SearXNG protect privacy?', + goal: 'Privacy design', + }); + assert.ok(evidence.length > 0); + assert.doesNotMatch(evidence.map((item) => item.claim).join('\n'), /data-hydro|HeaderMenu/); +}); + +test('Deep Search executes multi-round research and creates a cited report', async () => { + const { engine, store, searches } = createMockEngine(); + const started = engine.start({ question: '分析 AI Agent 市场', depth: 'standard', userId: 'user-1' }); + assert.deepEqual(started, { + taskId: 'task-complete', + status: 'queued', + service: 'tkmind-deep-search', + }); + const task = await waitFor(() => { + const current = engine.getTask(started.taskId); + return current.status === 'completed' ? current : null; + }); + assert.equal(task.progress, 100); + assert.equal(task.plan.length, 4); + assert.ok(searches.length >= 8); + assert.ok(task.sources.length > 0); + assert.match(task.report, /^# 分析 AI Agent 市场/m); + assert.match(task.report, /\[1\]/); + assert.match(task.report, /## 来源/); + assert.ok(task.events.some((event) => event.type === 'completed')); + assert.equal(store.listMemories({ userId: 'user-1' }).length, 1); + store.close(); +}); + +test('Deep Search uses a configured planner and citation-safe LLM report', async () => { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + let memoryPayload = null; + const llmReport = `# Model report + +This model-generated analysis is intentionally long enough to pass report validation and is grounded in the supplied evidence. It explains the finding, distinguishes evidence from inference, and preserves the required source citation. [1] + +## Sources + +1. Primary evidence [1]`; + const engine = createDeepSearchEngine({ + store, + idFactory: () => 'task-llm', + llmResolver: async ({ providerKeyId, model }) => { + assert.equal(providerKeyId, 'provider-key-1'); + assert.equal(model, 'research-model'); + return { + async plan() { + return { + research_plan: [{ + goal: 'Primary evidence', + queries: ['official evidence', 'independent evidence'], + sources: ['web'], + }], + }; + }, + async synthesize() { + return llmReport; + }, + }; + }, + memorySink: async (payload) => { + memoryPayload = payload; + }, + searchProvider: { + name: 'mock', + async search() { + return [{ + title: 'Primary source', + url: 'https://example.com/primary', + snippet: 'Primary evidence supports the tested claim.', + source: 'mock', + rank: 1, + }]; + }, + }, + reader: async (url) => ({ + url, + title: 'Primary source', + content: 'Primary evidence supports the tested claim with enough detail for a grounded research report and a verifiable citation.', + }), + }); + engine.start({ + question: 'Evaluate the tested claim', + depth: 'quick', + userId: 'memory-user', + llmProviderKeyId: 'provider-key-1', + llmModel: 'research-model', + }); + const task = await waitFor(() => { + const current = engine.getTask('task-llm'); + return current.status === 'completed' ? current : null; + }); + assert.equal(task.report, llmReport); + assert.equal(task.llmProviderKeyId, 'provider-key-1'); + assert.equal(task.llmModel, 'research-model'); + assert.ok(task.events.some((event) => event.type === 'llm_selected')); + assert.equal(memoryPayload.userId, 'memory-user'); + assert.equal(memoryPayload.taskId, 'task-llm'); + assert.ok(task.events.some((event) => event.type === 'memory_saved')); + store.close(); +}); + +test('Deep Search portal LLM gateway keeps provider secrets inside the portal', async () => { + let request = null; + const unauthorized = await executeDeepSearchLlmGateway({ + llmProviderService: {}, + expectedSecret: 'expected', + providedSecret: 'wrong', + input: {}, + }); + assert.equal(unauthorized.status, 401); + const authorized = await executeDeepSearchLlmGateway({ + expectedSecret: 'expected', + providedSecret: 'expected', + input: { + providerKeyId: 'key-1', + model: 'model-1', + messages: [{ role: 'user', content: 'Plan research' }], + }, + llmProviderService: { + async createChatCompletion(input) { + request = input; + return { + ok: true, + providerKeyId: 'key-1', + providerId: 'custom', + model: 'model-1', + reply: '{"research_plan":[]}', + }; + }, + }, + }); + assert.equal(authorized.status, 200); + assert.equal(request.providerKeyId, 'key-1'); + assert.equal(request.model, 'model-1'); + + const resolver = createPortalGatewayResearchLlmResolver({ + endpoint: 'http://portal.local/api/internal/deep-search/llm', + secret: 'expected', + fetchImpl: async (_url, options) => { + const body = JSON.parse(options.body); + assert.equal(options.headers.authorization, 'Bearer expected'); + assert.equal(body.providerKeyId, 'key-1'); + return new Response(JSON.stringify({ ok: true, reply: '{"research_plan":[]}' }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }, + }); + assert.equal(resolver.configured, true); + const resolved = await resolver({ providerKeyId: 'key-1', model: 'model-1' }); + assert.deepEqual(await resolved.plan('Question', 'quick'), { research_plan: [] }); +}); + +test('Deep Search supports cancellation while a provider is running', async () => { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + const engine = createDeepSearchEngine({ + store, + idFactory: () => 'task-cancel', + searchProvider: { + name: 'slow', + search(_query, { signal }) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve([]), 2000); + signal.addEventListener('abort', () => { + clearTimeout(timeout); + const error = new Error('cancelled'); + error.name = 'AbortError'; + reject(error); + }, { once: true }); + }); + }, + }, + }); + engine.start({ question: 'A long-running investigation', depth: 'deep' }); + await waitFor(() => engine.getTask('task-cancel').status === 'researching'); + const cancelling = engine.cancel('task-cancel'); + assert.equal(cancelling.status, 'cancelling'); + const cancelled = await waitFor(() => { + const current = engine.getTask('task-cancel'); + return current.status === 'cancelled' ? current : null; + }); + assert.equal(cancelled.phase, 'cancelled'); + store.close(); +}); + +test('Deep Search HTTP service exposes health, async start, status, search, and auth', async (t) => { + const { engine, store } = createMockEngine({ id: 'task-http' }); + let githubRequest; + const server = createDeepSearchHttpServer({ + engine, + secret: 'local-secret', + githubToken: 'github-secret-token', + githubSearch: async (query, options) => { + githubRequest = { query, options }; + return [{ + title: 'tkmind/code:server.mjs', + url: 'https://github.com/tkmind/code/blob/main/server.mjs', + snippet: 'server', + source: 'github', + rank: 1, + }]; + }, + logger: { warn() {} }, + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + store.close(); + }); + const address = server.address(); + const base = `http://127.0.0.1:${address.port}`; + const health = await fetch(`${base}/health`).then((response) => response.json()); + assert.equal(health.ok, true); + assert.equal(health.providers.github.configured, true); + assert.equal((await fetch(`${base}/v1/research`)).status, 401); + const headers = { 'content-type': 'application/json', 'x-secret-key': 'local-secret' }; + const github = await fetch(`${base}/v1/providers/github/search`, { + method: 'POST', + headers, + body: JSON.stringify({ query: 'repo:tkmind/code server', limit: 3 }), + }).then((response) => response.json()); + assert.equal(github.results.length, 1); + assert.equal(githubRequest.options.token, 'github-secret-token'); + assert.equal(githubRequest.options.limit, 3); + const search = await fetch(`${base}/v1/search`, { + method: 'POST', + headers, + body: JSON.stringify({ query: 'Deep Search', limit: 2 }), + }).then((response) => response.json()); + assert.equal(search.results.length, 2); + const started = await fetch(`${base}/v1/research`, { + method: 'POST', + headers: { ...headers, 'x-user-id': 'user-a' }, + body: JSON.stringify({ + question: 'Evaluate Deep Search', + depth: 'quick', + llm_provider_key_id: 'key-http', + llm_model: 'model-http', + }), + }).then((response) => response.json()); + assert.equal(started.task_id, 'task-http'); + assert.equal((await fetch(`${base}/v1/research/${started.task_id}`, { + headers: { ...headers, 'x-user-id': 'user-b' }, + })).status, 403); + const completed = await waitFor(async () => { + const response = await fetch(`${base}/v1/research/${started.task_id}`, { + headers: { ...headers, 'x-user-id': 'user-a' }, + }); + const task = await response.json(); + return task.status === 'completed' ? task : null; + }); + assert.equal(completed.progress, 100); + assert.equal(completed.llmProviderKeyId, 'key-http'); + assert.equal(completed.llmModel, 'model-http'); + assert.ok(completed.sources.every((source) => !Object.hasOwn(source, 'content'))); +}); diff --git a/docs/deep-search-service.md b/docs/deep-search-service.md new file mode 100644 index 0000000..ea062b8 --- /dev/null +++ b/docs/deep-search-service.md @@ -0,0 +1,152 @@ +# TKMind Deep Search Service + +TKMind Deep Search is a local-first, independently deployable research orchestrator. MindSearch treats it as a `research-http` service, in the same way that SearXNG is registered as a search provider. + +The standalone runtime uses the built-in `node:sqlite` module and therefore requires a Node.js release that provides `DatabaseSync` (Node 22.5 or newer; the local verification used Node 26). + +## Runtime flow + +1. Validate and persist the task. +2. Build a depth-aware research plan. +3. Run bounded, multi-round searches through SearXNG. +4. Canonicalize, deduplicate, and rerank source URLs. +5. Resolve DNS and block private destinations before reading public pages. +6. Extract evidence and remove duplicate claims. +7. Generate a Markdown report with numbered citations. +8. Persist task state, sources, events, report, and research memory in SQLite. + +The planner and report writer use an optional model selected from TKMind's Unified Model Center. MindSearch stores only the Provider Key ID and model name. Deep Search calls a secret-protected Portal gateway, and the provider API key never leaves the Portal process. If the selected model is unavailable, deterministic planning and citation-safe extractive reporting keep the service operational. + +`createDeepSearchEngine` also accepts an explicit `memorySink` callback. It is disabled by default; an in-process TKMind integration can inject `memoryV2.write` without giving the standalone service an unrestricted outbound memory endpoint. Memory sink failures are fail-open and recorded as task events. + +## Local run + +```bash +TKMIND_DEEP_SEARCH_SEARXNG_URL=http://127.0.0.1:8080/search \ +TKMIND_DEEP_SEARCH_DB=.deep-search/research.sqlite \ +TKMIND_DEEP_SEARCH_SECRET=local-shared-secret \ +npm run dev:deep-search +``` + +The server listens on `127.0.0.1:20100` by default. + +On production host `103`, SearXNG is exposed at +`http://127.0.0.1:20080/search`. Goosed containers reach host services through +`host.docker.internal`; Portal rewrites loopback MindSearch service endpoints +only when constructing the MCP extension environment. + +## API + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/health` | Runtime health and task counts | +| `POST` | `/v1/search` | Synchronous normalized search | +| `POST` | `/v1/research` | Start an asynchronous research task | +| `GET` | `/v1/research` | List recent tasks | +| `GET` | `/v1/research/:id` | Get progress, evidence metadata, and report | +| `GET` | `/v1/research/:id/events` | Stream progress as server-sent events | +| `DELETE` | `/v1/research/:id` | Cancel a queued or running task | + +Start request: + +```json +{ + "question": "分析 AI Agent 市场", + "depth": "standard", + "userId": "optional-user-id" +} +``` + +Start response: + +```json +{ + "task_id": "uuid", + "status": "queued", + "service": "tkmind-deep-search" +} +``` + +## Configuration + +| Variable | Default | Meaning | +| --- | --- | --- | +| `TKMIND_DEEP_SEARCH_HOST` | `127.0.0.1` | Bind host | +| `TKMIND_DEEP_SEARCH_PORT` | `20100` | HTTP port | +| `TKMIND_DEEP_SEARCH_DB` | `.deep-search/research.sqlite` | SQLite database | +| `TKMIND_DEEP_SEARCH_SEARXNG_URL` | `http://127.0.0.1:8080/search` | Upstream search endpoint | +| `TKMIND_DEEP_SEARCH_SECRET` | empty | Optional `X-Secret-Key` required by non-health routes | +| `TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL` | `http://127.0.0.1:8081/api/internal/deep-search/llm` | Portal LLM gateway | +| `TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET` | `TKMIND_DEEP_SEARCH_SECRET` | Portal gateway bearer secret | +| `GITHUB_TOKEN` | empty | GitHub code-search token, stored only in the standalone service environment | + +## Isolation and safety + +- The default bind address is loopback only. +- An optional service secret protects all non-health endpoints. +- The same secret can authenticate the local service to the Portal LLM gateway. +- MCP forwards the current TKMind user ID as `X-User-Id`. +- Task status, event, and cancellation routes enforce user ownership when a user ID is present. +- Source reading rejects credentials, loopback, link-local, private, multicast, and private-DNS destinations. +- Redirect targets are resolved and validated again. +- Request bodies, result counts, source content, timeouts, and task depths are bounded. +- GitHub Code Search is proxied by Deep Search so the provider token is never + copied into goosed extension environments. + +## Production runtime on 103 + +Deep Search is released independently from Portal and `memindadm`: + +```bash +npm run build:deep-search-runtime +CONFIRMED_CI_SHA="$(git rev-parse HEAD)" \ + bash scripts/release-deep-search-runtime-prod.sh +``` + +The production layout is: + +```text +/Users/john/Project/deep-search-runtime/ +├── current -> releases// +├── releases// +├── shared/.env +├── data/research.sqlite +├── logs/deep-search.log +└── backups// +``` + +The release workflow requires a clean `main` that exactly matches +`origin/main`, a matching confirmed CI SHA, artifact checksum verification, +SQLite online backup, LaunchAgent rollback, host health/search probes, and a +Goose-container reachability probe. It never releases or restarts Portal or +`memindadm`. + +The installer creates a random service secret on first install. Add a newly +issued, least-privilege GitHub token to `shared/.env` only after revoking any +token that has appeared in chat or logs. + +## MindSearch registration + +The built-in disabled service is: + +```json +{ + "id": "deep-search", + "adapter": "research-http", + "endpoint": "http://127.0.0.1:20100", + "healthPath": "/health", + "enabled": false, + "llmProviderKeyId": "", + "llmModel": "" +} +``` + +In `memindadm` → `MindSearch` → `TKMind Deep Search`, select an optional active Provider configuration and one of its models. The list comes directly from Unified Model Center. The `测试 LLM` action checks that exact Provider/model combination before saving the MindSearch configuration. Selecting “不使用 LLM（确定性降级）” leaves both fields empty. + +After local verification, enable the service and MindSearch, then select `deep-search` for the `research` route. New Goose sessions receive: + +- `tkmind_research` +- `tkmind_research_status` +- `tkmind_research_cancel` + +Enabling or publishing this service to production is a separate release action and is not performed by local development or tests. diff --git a/mindsearch-config.mjs b/mindsearch-config.mjs index 3585043..021baf5 100644 --- a/mindsearch-config.mjs +++ b/mindsearch-config.mjs @@ -1,11 +1,70 @@ const CONFIG_TABLE = 'h5_mindsearch_config'; const CONFIG_SCOPE = 'global'; +const SERVICE_KINDS = new Set(['provider', 'reader', 'orchestrator']); +const SERVICE_ADAPTERS = new Set(['searxng', 'github', 'reader', 'research-http']); +const ROUTE_KEYS = ['web', 'news', 'code', 'read', 'research']; + +const BUILTIN_SERVICES = Object.freeze([ + { + id: 'searxng', + name: 'SearXNG', + kind: 'provider', + adapter: 'searxng', + capabilities: ['search.web', 'search.news'], + endpoint: '', + healthPath: '/config', + enabled: false, + timeoutMs: 8000, + priority: 100, + }, + { + id: 'github', + name: 'GitHub Code', + kind: 'provider', + adapter: 'github', + capabilities: ['search.code'], + endpoint: 'https://api.github.com/search/code', + healthPath: '', + enabled: false, + timeoutMs: 8000, + priority: 90, + }, + { + id: 'reader', + name: 'Safe Reader', + kind: 'reader', + adapter: 'reader', + capabilities: ['search.read'], + endpoint: '', + healthPath: '', + enabled: false, + timeoutMs: 8000, + priority: 80, + }, + { + id: 'deep-search', + name: 'TKMind Deep Search', + kind: 'orchestrator', + adapter: 'research-http', + capabilities: ['search.web', 'research.plan', 'research.execute'], + endpoint: 'http://127.0.0.1:20100', + healthPath: '/health', + enabled: false, + timeoutMs: 120000, + priority: 100, + llmProviderKeyId: '', + llmModel: '', + }, +]); + export const MINDSEARCH_DEFAULT_CONFIG = Object.freeze({ enabled: false, mode: 'off', providers: { searxng: false, github: false, reader: false }, settings: { searxngEndpoint: '', maxResults: 10, timeoutMs: 8000, readerMaxChars: 12000 }, + services: BUILTIN_SERVICES, + routes: { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: 'deep-search' }, }); const clone = (value) => JSON.parse(JSON.stringify(value)); @@ -15,22 +74,199 @@ const bool = (value, fallback = false) => { return /^(1|true|yes|on)$/i.test(String(value)); }; +const boundedNumber = (value, fallback, min, max) => + Math.max(min, Math.min(max, Number(value ?? fallback) || fallback)); + +export function isValidSearchServiceEndpoint(value, { allowEmpty = true } = {}) { + const endpoint = String(value ?? '').trim(); + if (!endpoint) return allowEmpty; + try { + const url = new URL(endpoint); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) return false; + return !['0.0.0.0', '::', '[::]', '169.254.169.254'].includes(url.hostname); + } catch { + return false; + } +} + +function normalizeCapabilities(value, fallback = []) { + const capabilities = Array.isArray(value) ? value : fallback; + return [...new Set(capabilities + .map((item) => String(item ?? '').trim()) + .filter((item) => /^[a-z][a-z0-9_.-]{1,63}$/.test(item)))] + .slice(0, 20); +} + +function normalizeService(input = {}, fallback = {}) { + const id = String(input.id ?? fallback.id ?? '').trim().toLowerCase(); + if (!/^[a-z0-9][a-z0-9._-]{0,63}$/.test(id)) return null; + const kind = SERVICE_KINDS.has(input.kind) ? input.kind : (fallback.kind ?? 'provider'); + const adapter = SERVICE_ADAPTERS.has(input.adapter) ? input.adapter : (fallback.adapter ?? 'searxng'); + const endpoint = String(input.endpoint ?? fallback.endpoint ?? '').trim(); + if (!isValidSearchServiceEndpoint(endpoint)) return null; + const healthPathValue = String(input.healthPath ?? fallback.healthPath ?? '').trim(); + const healthPath = healthPathValue && /^\/[^\s]{0,255}$/.test(healthPathValue) ? healthPathValue : ''; + const llmProviderKeyIdValue = String(input.llmProviderKeyId ?? fallback.llmProviderKeyId ?? '').trim(); + const llmProviderKeyId = /^[a-zA-Z0-9._:-]{1,128}$/.test(llmProviderKeyIdValue) + ? llmProviderKeyIdValue + : ''; + const llmModelValue = String(input.llmModel ?? fallback.llmModel ?? '').trim(); + const llmModel = llmProviderKeyId && !/[\u0000-\u001f\u007f]/.test(llmModelValue) + ? llmModelValue.slice(0, 200) + : ''; + return { + id, + name: String(input.name ?? fallback.name ?? id).trim().slice(0, 80) || id, + kind, + adapter, + capabilities: normalizeCapabilities(input.capabilities, fallback.capabilities), + endpoint, + healthPath, + enabled: bool(input.enabled, bool(fallback.enabled)), + timeoutMs: boundedNumber(input.timeoutMs, fallback.timeoutMs ?? 8000, 1000, 120000), + priority: boundedNumber(input.priority, fallback.priority ?? 100, 1, 1000), + llmProviderKeyId, + llmModel, + }; +} + +function builtinsFromLegacy(input = {}) { + const providers = input.providers ?? {}; + const settings = input.settings ?? {}; + return BUILTIN_SERVICES.map((service) => normalizeService({ + ...service, + enabled: bool(providers[service.adapter], service.enabled), + endpoint: service.adapter === 'searxng' + ? String(settings.searxngEndpoint ?? service.endpoint) + : service.endpoint, + timeoutMs: service.adapter === 'research-http' + ? service.timeoutMs + : (settings.timeoutMs ?? service.timeoutMs), + }, service)); +} + +function normalizeServices(input = {}) { + const source = Array.isArray(input.services) ? input.services : builtinsFromLegacy(input); + const seen = new Set(); + const services = []; + for (const item of source) { + const fallback = BUILTIN_SERVICES.find((candidate) => candidate.id === item?.id) ?? {}; + const service = normalizeService(item, fallback); + if (!service || seen.has(service.id)) continue; + seen.add(service.id); + services.push(service); + } + for (const builtin of builtinsFromLegacy(input)) { + if (!seen.has(builtin.id)) services.push(builtin); + } + return services.slice(0, 30); +} + +function normalizeRoutes(value, services) { + const ids = new Set(services.map((service) => service.id)); + const defaults = MINDSEARCH_DEFAULT_CONFIG.routes; + return Object.fromEntries(ROUTE_KEYS.map((key) => { + const requested = String(value?.[key] ?? defaults[key] ?? '').trim(); + return [key, !requested || ids.has(requested) ? requested : '']; + })); +} + +export function resolveMindSearchService(config, route) { + const normalized = normalizeMindSearchConfig(config); + const serviceId = normalized.routes?.[route] ?? ''; + return normalized.services.find((service) => service.id === serviceId && service.enabled) ?? null; +} + +export async function probeMindSearchService(service, { fetchImpl = fetch } = {}) { + const normalized = normalizeService(service); + if (!normalized) throw new Error('Invalid search service configuration'); + if (!normalized.endpoint) { + return { + ok: normalized.adapter === 'reader', + serviceId: normalized.id, + adapter: normalized.adapter, + status: null, + latencyMs: 0, + message: normalized.adapter === 'reader' ? 'Built-in reader is available' : 'Service endpoint is not configured', + }; + } + const startedAt = Date.now(); + let url; + if (normalized.adapter === 'searxng') { + url = new URL(normalized.endpoint); + url.searchParams.set('q', 'OpenAI'); + url.searchParams.set('format', 'json'); + } else if (normalized.healthPath) { + url = new URL(normalized.healthPath, normalized.endpoint); + } else { + return { + ok: true, + serviceId: normalized.id, + adapter: normalized.adapter, + status: null, + latencyMs: 0, + message: 'Configuration is valid; no health path is configured', + }; + } + try { + const response = await fetchImpl(url, { + signal: AbortSignal.timeout(normalized.timeoutMs), + headers: { + accept: 'application/json,text/plain', + ...(normalized.adapter === 'research-http' && process.env.TKMIND_DEEP_SEARCH_SECRET + ? { 'x-secret-key': process.env.TKMIND_DEEP_SEARCH_SECRET } + : {}), + }, + }); + let resultCount = null; + if (normalized.adapter === 'searxng' && response.ok) { + const body = await response.json(); + resultCount = Array.isArray(body.results) ? body.results.length : 0; + } + return { + ok: response.ok && (resultCount == null || resultCount > 0), + serviceId: normalized.id, + adapter: normalized.adapter, + status: response.status, + latencyMs: Date.now() - startedAt, + resultCount, + message: response.ok ? 'Health check completed' : `Service returned HTTP ${response.status}`, + }; + } catch (error) { + return { + ok: false, + serviceId: normalized.id, + adapter: normalized.adapter, + status: null, + latencyMs: Date.now() - startedAt, + message: error instanceof Error ? error.message : 'Service health check failed', + }; + } +} + export function normalizeMindSearchConfig(input = {}) { const mode = ['off', 'shadow', 'assist'].includes(input.mode) ? input.mode : 'off'; + const services = normalizeServices(input); + const serviceByAdapter = (adapter) => services.find((service) => service.adapter === adapter); + const searxng = serviceByAdapter('searxng'); + const github = serviceByAdapter('github'); + const reader = serviceByAdapter('reader'); const config = { enabled: bool(input.enabled), mode, providers: { - searxng: bool(input.providers?.searxng), - github: bool(input.providers?.github), - reader: bool(input.providers?.reader), + searxng: bool(searxng?.enabled), + github: bool(github?.enabled), + reader: bool(reader?.enabled), }, settings: { - searxngEndpoint: String(input.settings?.searxngEndpoint ?? '').trim(), - maxResults: Math.max(1, Math.min(20, Number(input.settings?.maxResults ?? 10) || 10)), - timeoutMs: Math.max(1000, Math.min(30000, Number(input.settings?.timeoutMs ?? 8000) || 8000)), - readerMaxChars: Math.max(1000, Math.min(50000, Number(input.settings?.readerMaxChars ?? 12000) || 12000)), + searxngEndpoint: String(searxng?.endpoint ?? input.settings?.searxngEndpoint ?? '').trim(), + maxResults: boundedNumber(input.settings?.maxResults, 10, 1, 20), + timeoutMs: boundedNumber(input.settings?.timeoutMs, 8000, 1000, 30000), + readerMaxChars: boundedNumber(input.settings?.readerMaxChars, 12000, 1000, 50000), }, + services, + routes: normalizeRoutes(input.routes, services), }; if (!config.enabled || config.mode === 'off') { config.enabled = false; @@ -49,6 +285,57 @@ function envConfig(env = process.env) { github: env.TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED, reader: env.TKMIND_SEARCH_READER_ENABLED, }, + settings: { + searxngEndpoint: env.TKMIND_SEARCH_SEARXNG_URL, + maxResults: env.TKMIND_SEARCH_MAX_RESULTS, + timeoutMs: env.TKMIND_SEARCH_TIMEOUT_MS, + readerMaxChars: env.TKMIND_SEARCH_READER_MAX_CHARS, + }, + services: parseJsonArray(env.TKMIND_SEARCH_SERVICES_JSON), + routes: parseJsonObject(env.TKMIND_SEARCH_ROUTES_JSON), + }); +} + +function parseJsonArray(value) { + if (!value) return undefined; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function parseJsonObject(value) { + if (!value) return undefined; + try { + const parsed = JSON.parse(value); + return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function applyLegacyPatchToServices(current, patch) { + if (Array.isArray(patch?.services)) return patch.services; + const providerPatch = patch?.providers ?? {}; + const settingPatch = patch?.settings ?? {}; + return current.services.map((service) => { + if (service.adapter === 'searxng') { + return { + ...service, + ...(providerPatch.searxng == null ? {} : { enabled: providerPatch.searxng }), + ...(settingPatch.searxngEndpoint == null ? {} : { endpoint: settingPatch.searxngEndpoint }), + ...(settingPatch.timeoutMs == null ? {} : { timeoutMs: settingPatch.timeoutMs }), + }; + } + if (service.adapter === 'github' && providerPatch.github != null) { + return { ...service, enabled: providerPatch.github }; + } + if (service.adapter === 'reader' && providerPatch.reader != null) { + return { ...service, enabled: providerPatch.reader }; + } + return service; }); } @@ -61,7 +348,7 @@ export async function ensureMindSearchConfigSchema(pool) { ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`); } -export function createMindSearchConfigService(pool, { env = process.env } = {}) { +export function createMindSearchConfigService(pool, { env = process.env, fetchImpl = fetch } = {}) { let ready = null; const ensure = async () => { if (!pool) return; @@ -82,9 +369,22 @@ export function createMindSearchConfigService(pool, { env = process.env } = {}) ensureSchema: ensure, getEffectiveConfig: async () => (await load()).config, getAdminConfig: async () => load(), + testService: async (serviceId) => { + const current = await load(); + const service = current.config.services.find((item) => item.id === String(serviceId ?? '').trim()); + if (!service) throw Object.assign(new Error('Search service not found'), { code: 'SEARCH_SERVICE_NOT_FOUND' }); + return probeMindSearchService(service, { fetchImpl }); + }, updateAdminConfig: async (patch, { updatedBy = null } = {}) => { const current = await load(); - const next = normalizeMindSearchConfig({ ...current.config, ...patch, providers: { ...current.config.providers, ...(patch?.providers ?? {}) }, settings: { ...current.config.settings, ...(patch?.settings ?? {}) } }); + const next = normalizeMindSearchConfig({ + ...current.config, + ...patch, + providers: { ...current.config.providers, ...(patch?.providers ?? {}) }, + settings: { ...current.config.settings, ...(patch?.settings ?? {}) }, + services: applyLegacyPatchToServices(current.config, patch), + routes: { ...current.config.routes, ...(patch?.routes ?? {}) }, + }); if (!pool) return { config: next, source: 'env' }; await ensure(); const now = Date.now(); @@ -93,7 +393,16 @@ export function createMindSearchConfigService(pool, { env = process.env } = {}) }, getRuntimeState: async () => { const config = await (async () => (await load()).config)(); - return { enabled: config.enabled, mode: config.mode, providers: config.providers, effective: config.enabled && config.mode !== 'off' }; + return { + enabled: config.enabled, + mode: config.mode, + providers: config.providers, + services: config.services.map(({ id, name, kind, adapter, capabilities, enabled, timeoutMs, priority }) => ({ + id, name, kind, adapter, capabilities, enabled, timeoutMs, priority, + })), + routes: config.routes, + effective: config.enabled && config.mode !== 'off', + }; }, }; } diff --git a/mindsearch-providers.mjs b/mindsearch-providers.mjs index 3ef66b3..b669d39 100644 --- a/mindsearch-providers.mjs +++ b/mindsearch-providers.mjs @@ -1,10 +1,21 @@ import { isSafeHttpUrl, normalizeSearchResult } from './search-capability.mjs'; +import { isValidSearchServiceEndpoint } from './mindsearch-config.mjs'; -export async function searchSearxng(query, { limit = 10, endpoint = process.env.TKMIND_SEARCH_SEARXNG_URL, fetchImpl = fetch } = {}) { +export async function searchSearxng(query, { + limit = 10, + endpoint = process.env.TKMIND_SEARCH_SEARXNG_URL, + fetchImpl = fetch, + timeoutMs = Number(process.env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000), + signal, +} = {}) { if (!endpoint) throw new Error('SearXNG endpoint is not configured'); try { if (!['http:', 'https:'].includes(new URL(endpoint).protocol)) throw new Error('unsupported protocol'); } catch { throw new Error('SearXNG endpoint is not configured safely'); } const url = new URL(endpoint); url.searchParams.set('q', query); url.searchParams.set('format', 'json'); - const response = await fetchImpl(url, { signal: AbortSignal.timeout(Number(process.env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000)), headers: { accept: 'application/json' } }); + const timeout = AbortSignal.timeout(timeoutMs); + const response = await fetchImpl(url, { + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + headers: { accept: 'application/json' }, + }); if (!response.ok) throw new Error(`SearXNG returned ${response.status}`); const body = await response.json(); return (Array.isArray(body.results) ? body.results : []).slice(0, limit).map((item, index) => normalizeSearchResult({ ...item, source: 'searxng' }, index)); @@ -29,3 +40,136 @@ export async function searchGithubCode(query, { limit = 10, token = process.env. const body = await response.json(); return (Array.isArray(body.items) ? body.items : []).slice(0, limit).map((item, index) => normalizeSearchResult({ title: `${item.repository?.full_name ?? ''}:${item.path ?? item.name ?? ''}`, url: item.html_url, snippet: item.repository?.description ?? '', source: 'github' }, index)); } + +function serviceUrl(endpoint, path) { + if (!isValidSearchServiceEndpoint(endpoint, { allowEmpty: false })) throw new Error('Search service endpoint is not configured safely'); + const base = endpoint.endsWith('/') ? endpoint : `${endpoint}/`; + return new URL(path.replace(/^\//, ''), base); +} + +async function postServiceJson(endpoint, path, payload, { + timeoutMs = 30000, + fetchImpl = fetch, + userId = null, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const response = await fetchImpl(serviceUrl(endpoint, path), { + method: 'POST', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + 'content-type': 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, + body: JSON.stringify(payload), + }); + if (!response.ok) throw new Error(`Search service returned ${response.status}`); + return response.json(); +} + +export async function searchGithubGateway(query, { + endpoint = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL, + limit = 10, + timeoutMs = 12000, + fetchImpl = fetch, + secret = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET + || process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const body = await postServiceJson(endpoint, '/v1/providers/github/search', { + query, + limit, + }, { + timeoutMs, + fetchImpl, + secret, + }); + return (Array.isArray(body.results) ? body.results : []) + .slice(0, limit) + .map((item, index) => normalizeSearchResult({ ...item, source: 'github' }, index)); +} + +export async function searchResearchService(query, { + endpoint, + type = 'web', + limit = 10, + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const body = await postServiceJson(endpoint, '/v1/search', { query, type, limit }, { + timeoutMs, + fetchImpl, + secret, + }); + return (Array.isArray(body.results) ? body.results : []) + .slice(0, limit) + .map((item, index) => normalizeSearchResult({ ...item, source: item.source || 'research-service' }, index)); +} + +export async function startResearchTask(question, { + endpoint, + depth = 'standard', + userId = null, + llmProviderKeyId = '', + llmModel = '', + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const body = await postServiceJson(endpoint, '/v1/research', { + question, + depth, + ...(userId ? { userId } : {}), + ...(llmProviderKeyId && llmModel + ? { llm_provider_key_id: llmProviderKeyId, llm_model: llmModel } + : {}), + }, { timeoutMs, fetchImpl, userId, secret }); + if (!body || typeof body !== 'object' || !body.task_id) throw new Error('Research service returned an invalid task'); + return { + taskId: String(body.task_id), + status: String(body.status ?? 'running'), + service: String(body.service ?? 'research-service'), + }; +} + +export async function getResearchTask(taskId, { + endpoint, + userId = null, + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), { + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, + }); + if (response.status === 404) throw new Error('Research task was not found'); + if (!response.ok) throw new Error(`Search service returned ${response.status}`); + return response.json(); +} + +export async function cancelResearchTask(taskId, { + endpoint, + userId = null, + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), { + method: 'DELETE', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, + }); + if (response.status === 404) throw new Error('Research task was not found'); + if (!response.ok) throw new Error(`Search service returned ${response.status}`); + return response.json(); +} diff --git a/mindsearch.test.mjs b/mindsearch.test.mjs index 18dcae1..3f20559 100644 --- a/mindsearch.test.mjs +++ b/mindsearch.test.mjs @@ -1,13 +1,85 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { buildCitations, isSafeHttpUrl, validateSearchRequest } from './search-capability.mjs'; -import { MINDSEARCH_DEFAULT_CONFIG, normalizeMindSearchConfig } from './mindsearch-config.mjs'; -import { buildAgentExtensionPolicy, DEFAULT_USER_CAPABILITIES } from './capabilities.mjs'; -import { searchGithubCode, searchSearxng } from './mindsearch-providers.mjs'; +import { + MINDSEARCH_DEFAULT_CONFIG, + normalizeMindSearchConfig, + probeMindSearchService, + resolveMindSearchService, +} from './mindsearch-config.mjs'; +import { + buildAgentExtensionPolicy, + DEFAULT_USER_CAPABILITIES, +} from './capabilities.mjs'; +import { + cancelResearchTask, + getResearchTask, + searchGithubCode, + searchGithubGateway, + searchResearchService, + searchSearxng, + startResearchTask, +} from './mindsearch-providers.mjs'; test('MindSearch defaults to disabled and normalizes unsafe modes', () => { assert.equal(MINDSEARCH_DEFAULT_CONFIG.enabled, false); - assert.deepEqual(normalizeMindSearchConfig({ enabled: true, mode: 'invalid', providers: { github: true } }), { enabled: false, mode: 'off', providers: { searxng: false, github: true, reader: false }, settings: { searxngEndpoint: '', maxResults: 10, timeoutMs: 8000, readerMaxChars: 12000 } }); + const config = normalizeMindSearchConfig({ enabled: true, mode: 'invalid', providers: { github: true } }); + assert.equal(config.enabled, false); + assert.equal(config.mode, 'off'); + assert.deepEqual(config.providers, { searxng: false, github: true, reader: false }); + assert.equal(config.services.find((service) => service.id === 'github').enabled, true); + assert.deepEqual(config.routes, { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: 'deep-search' }); + assert.equal(config.services.find((service) => service.id === 'deep-search').enabled, false); +}); + +test('MindSearch service registry accepts research orchestrators and rejects unsafe endpoints', () => { + const config = normalizeMindSearchConfig({ + enabled: true, + mode: 'assist', + services: [ + { + id: 'research-main', + name: 'Research', + kind: 'orchestrator', + adapter: 'research-http', + endpoint: 'http://127.0.0.1:20100', + healthPath: '/health', + capabilities: ['search.web', 'research.execute', 'research.execute', 'INVALID CAPABILITY'], + enabled: true, + timeoutMs: 500000, + llmProviderKeyId: 'provider-key-1', + llmModel: 'research-model', + }, + { id: 'metadata', adapter: 'research-http', endpoint: 'http://169.254.169.254/latest', enabled: true }, + ], + routes: { research: 'research-main', web: 'research-main' }, + }); + assert.equal(config.services.some((service) => service.id === 'metadata'), false); + assert.deepEqual(config.services.find((service) => service.id === 'research-main').capabilities, ['search.web', 'research.execute']); + assert.equal(config.services.find((service) => service.id === 'research-main').timeoutMs, 120000); + assert.equal(config.services.find((service) => service.id === 'research-main').llmProviderKeyId, 'provider-key-1'); + assert.equal(config.services.find((service) => service.id === 'research-main').llmModel, 'research-model'); + assert.equal(resolveMindSearchService(config, 'research').id, 'research-main'); +}); + +test('MindSearch service probe supports loopback SearXNG without exposing arbitrary response data', async () => { + let requested; + const result = await probeMindSearchService({ + id: 'searxng-local', + name: 'Local', + kind: 'provider', + adapter: 'searxng', + endpoint: 'http://127.0.0.1:8080/search', + enabled: true, + }, { + fetchImpl: async (url) => { + requested = String(url); + return { ok: true, status: 200, json: async () => ({ results: [{ title: 'A' }] }) }; + }, + }); + assert.match(requested, /q=OpenAI/); + assert.equal(result.ok, true); + assert.equal(result.resultCount, 1); }); test('search request validation and citations are deterministic', () => { @@ -28,6 +100,48 @@ test('MindSearch never changes legacy web extension and is gated by capability/c assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'tkmind-search'), false); policy = buildAgentExtensionPolicy({ ...base, search_external: true }, { mindSearchConfig: { enabled: true, mode: 'assist', providers: {} } }); assert.ok(policy.extensionOverrides.some((ext) => ext.name === 'tkmind-search')); + policy = buildAgentExtensionPolicy({ ...base, search_external: true }, { + userId: 'user-123', + mindSearchConfig: { + enabled: true, + mode: 'assist', + providers: {}, + settings: { searxngEndpoint: 'http://127.0.0.1:20080/search' }, + services: [ + { + id: 'searxng', + enabled: true, + adapter: 'searxng', + endpoint: 'http://127.0.0.1:20080/search', + }, + { + id: 'deep-search', + enabled: true, + adapter: 'research-http', + endpoint: 'http://127.0.0.1:20100', + }, + ], + routes: { research: 'deep-search' }, + }, + }); + const extension = policy.extensionOverrides.find((ext) => ext.name === 'tkmind-search'); + assert.equal(extension.envs.TKMIND_SEARCH_USER_ID, 'user-123'); + assert.equal( + extension.envs.TKMIND_SEARCH_SEARXNG_URL, + 'http://host.docker.internal:20080/search', + ); + assert.equal( + extension.envs.TKMIND_SEARCH_GITHUB_GATEWAY_URL, + 'http://host.docker.internal:20100/', + ); + assert.equal(extension.envs.GITHUB_TOKEN, undefined); + const mcpServices = JSON.parse(extension.envs.TKMIND_SEARCH_SERVICES_JSON); + assert.equal( + mcpServices.find((service) => service.id === 'deep-search').endpoint, + 'http://host.docker.internal:20100/', + ); + assert.ok(extension.available_tools.includes('tkmind_research_status')); + assert.ok(extension.available_tools.includes('tkmind_research_cancel')); }); test('SearXNG adapter normalizes provider results without requiring a live network', async () => { @@ -42,3 +156,77 @@ test('GitHub code adapter sends bounded queries and normalizes results', async ( assert.equal(requested.options.headers.accept, 'application/vnd.github+json'); assert.equal(result[0].source, 'github'); }); + +test('GitHub gateway keeps the provider token outside the MCP process', async () => { + let requested; + const result = await searchGithubGateway('repo:openai goose', { + endpoint: 'http://deep-search.local:20100', + secret: 'gateway-secret', + fetchImpl: async (url, options) => { + requested = { url: String(url), options }; + return { + ok: true, + json: async () => ({ + results: [{ + title: 'openai/goose:src/goose.rs', + url: 'https://github.com/openai/goose/blob/main/src/goose.rs', + snippet: 'agent', + }], + }), + }; + }, + }); + assert.equal(requested.url, 'http://deep-search.local:20100/v1/providers/github/search'); + assert.equal(requested.options.headers['x-secret-key'], 'gateway-secret'); + assert.equal(requested.options.headers.authorization, undefined); + assert.equal(result[0].source, 'github'); +}); + +test('Research service adapter uses the stable HTTP contract', async () => { + const requests = []; + const fetchImpl = async (url, options) => { + requests.push({ url: String(url), options }); + if (String(url).endsWith('/v1/research')) { + return { ok: true, json: async () => ({ task_id: 'task-1', status: 'running' }) }; + } + if (String(url).endsWith('/v1/research/task-1')) { + return { + ok: true, + status: 200, + json: async () => ({ + id: 'task-1', + status: options?.method === 'DELETE' ? 'cancelling' : 'researching', + progress: 55, + }), + }; + } + return { ok: true, json: async () => ({ results: [{ title: 'Report', url: 'https://example.com/report', snippet: 'evidence' }] }) }; + }; + const results = await searchResearchService('AI Agent', { endpoint: 'http://research.local', limit: 5, fetchImpl }); + const task = await startResearchTask('Analyze AI Agent', { + endpoint: 'http://research.local', + depth: 'deep', + userId: 'user-1', + llmProviderKeyId: 'provider-key-1', + llmModel: 'research-model', + secret: 'secret-1', + fetchImpl, + }); + const status = await getResearchTask('task-1', { endpoint: 'http://research.local', fetchImpl }); + const cancelled = await cancelResearchTask('task-1', { endpoint: 'http://research.local', fetchImpl }); + assert.equal(results[0].source, 'research-service'); + assert.deepEqual(task, { taskId: 'task-1', status: 'running', service: 'research-service' }); + assert.equal(status.status, 'researching'); + assert.equal(cancelled.status, 'cancelling'); + assert.equal(JSON.parse(requests[0].options.body).query, 'AI Agent'); + assert.deepEqual(JSON.parse(requests[1].options.body), { + question: 'Analyze AI Agent', + depth: 'deep', + userId: 'user-1', + llm_provider_key_id: 'provider-key-1', + llm_model: 'research-model', + }); + assert.equal(requests[1].options.headers['x-user-id'], 'user-1'); + assert.equal(requests[1].options.headers['x-secret-key'], 'secret-1'); + assert.equal(requests[3].options.method, 'DELETE'); +}); diff --git a/package.json b/package.json index 9733ec8..a91fff2 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dev:vite": "vite", "dev:server": "node server.mjs", "dev:adm": "node admin-server.mjs", + "dev:deep-search": "node deep-search-server.mjs", "dev:plaza-express": "node plaza-server.mjs", "dev:ops": "cd ops && npm run dev", "launch:executor": "node scripts/launch-executor.mjs", @@ -35,6 +36,7 @@ "backfill:plaza-public-urls": "node scripts/backfill-plaza-public-urls.mjs", "enrich:plaza": "node scripts/enrich-plaza-posts.mjs", "build": "vite build", + "build:deep-search-runtime": "node scripts/build-deep-search-runtime.mjs", "build:portal-runtime": "node scripts/build-portal-runtime.mjs", "build:imgproxy-runtime": "bash scripts/build-imgproxy-runtime-image.sh", "build:mindspace-service-runtime": "node scripts/build-mindspace-service-runtime.mjs", @@ -61,6 +63,7 @@ "test:scenario:john4-diet": "node scripts/run-scenario-test.mjs --scenario john4-children-hobby-diet-update", "test": "node --test auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", + "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", "test:mindspace-service": "node --test mindspace-service/mindspace-rpc-server.test.mjs", "verify:chat-finish-sync": "node scripts/verify-chat-finish-sync.mjs", diff --git a/scripts/build-deep-search-runtime.mjs b/scripts/build-deep-search-runtime.mjs new file mode 100644 index 0000000..829b953 --- /dev/null +++ b/scripts/build-deep-search-runtime.mjs @@ -0,0 +1,75 @@ +import { spawn } from 'node:child_process'; +import { + chmod, + copyFile, + mkdir, + rm, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const runtimeRoot = path.join(root, '.runtime', 'deep-search'); +const esbuildBin = path.join(root, 'node_modules', '.bin', 'esbuild'); +const runtimeNodeTarget = process.env.DEEP_SEARCH_RUNTIME_NODE_TARGET || 'node24'; + +function run(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: root, + stdio: 'inherit', + env: process.env, + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) return resolve(); + reject(new Error(`${command} exited with ${code ?? signal}`)); + }); + }); +} + +await rm(runtimeRoot, { recursive: true, force: true }); +await mkdir(path.join(runtimeRoot, 'scripts'), { recursive: true }); + +await run(esbuildBin, [ + 'deep-search-server.mjs', + '--bundle', + '--platform=node', + '--format=esm', + `--target=${runtimeNodeTarget}`, + '--outfile=.runtime/deep-search/deep-search-server.mjs', + '--banner:js=import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', +]); + +for (const script of [ + 'run-deep-search-prod.sh', + 'install-deep-search-runtime-prod.sh', +]) { + const source = path.join(root, 'scripts', script); + const target = path.join(runtimeRoot, 'scripts', script); + await copyFile(source, target); + await chmod(target, 0o755); +} + +await writeFile( + path.join(runtimeRoot, 'RUNBOOK.txt'), + [ + 'TKMind Deep Search standalone runtime', + '', + 'Production base: /Users/john/Project/deep-search-runtime', + 'Persistent env: shared/.env', + 'Persistent data: data/research.sqlite', + 'Release code: releases//', + 'Active release: current -> releases//', + 'LaunchAgent: cn.tkmind.memind-deep-search', + 'Health: http://127.0.0.1:20100/health', + 'SearXNG: http://127.0.0.1:20080/search', + '', + 'The GitHub token stays in shared/.env and is never passed to goosed MCP processes.', + 'Portal and memindadm releases are intentionally separate.', + '', + ].join('\n'), +); + +console.log(`Deep Search runtime built at ${runtimeRoot}`); diff --git a/scripts/deep-search-runtime-package.test.mjs b/scripts/deep-search-runtime-package.test.mjs new file mode 100644 index 0000000..fbd50ad --- /dev/null +++ b/scripts/deep-search-runtime-package.test.mjs @@ -0,0 +1,31 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const read = (relativePath) => readFileSync(path.join(root, relativePath), 'utf8'); + +test('Deep Search production runtime is isolated from Portal releases', () => { + const builder = read('scripts/build-deep-search-runtime.mjs'); + assert.match(builder, /deep-search-server\.mjs/); + assert.match(builder, /\.runtime', 'deep-search/); + + const runner = read('scripts/run-deep-search-prod.sh'); + assert.match(runner, /shared\/\.env/); + assert.match(runner, /data\/research\.sqlite/); + assert.match(runner, /127\.0\.0\.1:20080\/search/); + + const installer = read('scripts/install-deep-search-runtime-prod.sh'); + assert.match(installer, /cn\.tkmind\.memind-deep-search/); + assert.match(installer, /chmod 600/); + assert.match(installer, /openssl rand -hex 32/); + + const release = read('scripts/release-deep-search-runtime-prod.sh'); + assert.match(release, /branch.*main/); + assert.match(release, /CONFIRMED_CI_SHA/); + assert.match(release, /shasum -a 256/); + assert.match(release, /rollback/); + assert.doesNotMatch(release, /release-portal-runtime-prod/); +}); diff --git a/scripts/install-deep-search-runtime-prod.sh b/scripts/install-deep-search-runtime-prod.sh new file mode 100644 index 0000000..9368483 --- /dev/null +++ b/scripts/install-deep-search-runtime-prod.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Runs on 103 from a verified Deep Search release directory. +set -euo pipefail + +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}" +RELEASE_DIR="${DEEP_SEARCH_RELEASE_DIR:?DEEP_SEARCH_RELEASE_DIR is required}" +ENV_DIR="${RUNTIME_BASE}/shared" +ENV_FILE="${ENV_DIR}/.env" +DATA_DIR="${RUNTIME_BASE}/data" +LOG_DIR="${RUNTIME_BASE}/logs" +PLIST="${HOME}/Library/LaunchAgents/cn.tkmind.memind-deep-search.plist" +LABEL="cn.tkmind.memind-deep-search" +GUI="gui/$(id -u)" +NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}" + +[[ -x "${NODE_BIN}" ]] || { + echo "missing Node.js runtime: ${NODE_BIN}" >&2 + exit 1 +} +[[ -f "${RELEASE_DIR}/deep-search-server.mjs" ]] || { + echo "missing Deep Search server in ${RELEASE_DIR}" >&2 + exit 1 +} + +mkdir -p "${ENV_DIR}" "${DATA_DIR}" "${LOG_DIR}" "$(dirname "${PLIST}")" +touch "${ENV_FILE}" +chmod 600 "${ENV_FILE}" + +ensure_env() { + local key="$1" + local value="$2" + if ! grep -qE "^${key}=" "${ENV_FILE}"; then + printf '%s=%s\n' "${key}" "${value}" >> "${ENV_FILE}" + fi +} + +if ! grep -qE '^TKMIND_DEEP_SEARCH_SECRET=.+$' "${ENV_FILE}"; then + secret="$(openssl rand -hex 32)" + ensure_env TKMIND_DEEP_SEARCH_SECRET "${secret}" +fi +ensure_env TKMIND_DEEP_SEARCH_HOST "127.0.0.1" +ensure_env TKMIND_DEEP_SEARCH_PORT "20100" +ensure_env TKMIND_DEEP_SEARCH_DB "${DATA_DIR}/research.sqlite" +ensure_env TKMIND_DEEP_SEARCH_SEARXNG_URL "http://127.0.0.1:20080/search" +ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL "http://127.0.0.1:8081/api/internal/deep-search/llm" +ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET '${TKMIND_DEEP_SEARCH_SECRET}' + +ln -sfn "${RELEASE_DIR}" "${RUNTIME_BASE}/current" + +cat > "${PLIST}" < + + +Label${LABEL} +ProgramArguments${RUNTIME_BASE}/current/scripts/run-deep-search-prod.sh +WorkingDirectory${RUNTIME_BASE}/current +EnvironmentVariables + DEEP_SEARCH_RUNTIME_BASE${RUNTIME_BASE} + NODE_BIN${NODE_BIN} + +RunAtLoad +KeepAlive +ProcessTypeBackground +ThrottleInterval10 +StandardOutPath${LOG_DIR}/deep-search.log +StandardErrorPath${LOG_DIR}/deep-search.log + +EOF + +plutil -lint "${PLIST}" >/dev/null +launchctl bootout "${GUI}/${LABEL}" >/dev/null 2>&1 || true +launchctl bootstrap "${GUI}" "${PLIST}" +launchctl enable "${GUI}/${LABEL}" +launchctl kickstart -k "${GUI}/${LABEL}" + +for _ in $(seq 1 30); do + if curl -fsS --max-time 2 "http://127.0.0.1:20100/health" >/dev/null; then + exit 0 + fi + sleep 1 +done + +echo "Deep Search health check timed out" >&2 +exit 1 diff --git a/scripts/release-deep-search-runtime-prod.sh b/scripts/release-deep-search-runtime-prod.sh new file mode 100644 index 0000000..142d2cb --- /dev/null +++ b/scripts/release-deep-search-runtime-prod.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Publishes only the independently versioned Deep Search runtime to 103. +# Portal and memindadm are not modified by this workflow. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOST="${STUDIO_HOST:-58.38.22.103}" +REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}" +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-${REMOTE_ROOT}/deep-search-runtime}" +INCOMING="${REMOTE_ROOT}/incoming/deep-search-runtime" +RELEASE_ID="$(date +%Y%m%d-%H%M%S)-$(git -C "${ROOT}" rev-parse --short HEAD)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/deep-search-runtime-release.XXXXXX")" +DRY_RUN=0 +AUTO_YES=0 + +cleanup() { + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=1 ;; + --yes) AUTO_YES=1 ;; + -h|--help) + cat <<'EOF' +Usage: bash scripts/release-deep-search-runtime-prod.sh [--dry-run] [--yes] + +Required production gates: + - current branch is main + - HEAD exactly matches origin/main + - worktree is clean + - CONFIRMED_CI_SHA equals the full current HEAD + +This installs only Deep Search on 103. It does not release Portal or memindadm. +EOF + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + exit 1 + ;; + esac + shift +done + +git -C "${ROOT}" fetch origin main --quiet +branch="$(git -C "${ROOT}" branch --show-current)" +head_sha="$(git -C "${ROOT}" rev-parse HEAD)" +origin_sha="$(git -C "${ROOT}" rev-parse origin/main)" +[[ "${branch}" == "main" ]] || { + echo "Deep Search production release must run from main" >&2 + exit 1 +} +[[ "${head_sha}" == "${origin_sha}" ]] || { + echo "main must exactly match origin/main" >&2 + exit 1 +} +[[ -z "$(git -C "${ROOT}" status --porcelain=v1 --untracked-files=all)" ]] || { + echo "worktree must be clean" >&2 + exit 1 +} +[[ "${CONFIRMED_CI_SHA:-}" == "${head_sha}" ]] || { + echo "CONFIRMED_CI_SHA must equal the verified main HEAD: ${head_sha}" >&2 + exit 1 +} +ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh" + +( + cd "${ROOT}" + npm run test:deep-search + node --test capabilities.test.mjs scripts/deep-search-runtime-package.test.mjs + npm run build:deep-search-runtime +) + +RUNTIME_DIR="${ROOT}/.runtime/deep-search" +for file in \ + deep-search-server.mjs \ + scripts/run-deep-search-prod.sh \ + scripts/install-deep-search-runtime-prod.sh \ + RUNBOOK.txt; do + [[ -f "${RUNTIME_DIR}/${file}" ]] || { + echo "missing runtime artifact: ${file}" >&2 + exit 1 + } +done + +MANIFEST="${RUNTIME_DIR}/release-manifest.txt" +{ + echo "release_id=${RELEASE_ID}" + echo "git_head=${head_sha}" + echo "git_branch=${branch}" + echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')" + echo "artifact=.runtime/deep-search" + echo "persistent_env=${RUNTIME_BASE}/shared/.env" + echo "persistent_db=${RUNTIME_BASE}/data/research.sqlite" +} > "${MANIFEST}" + +ARCHIVE="${TMP_DIR}/deep-search-runtime-${RELEASE_ID}.tar.gz" +SHA_FILE="${ARCHIVE}.sha256" +tar -C "${RUNTIME_DIR}" -czf "${ARCHIVE}" . +(cd "${TMP_DIR}" && shasum -a 256 "$(basename "${ARCHIVE}")" > "$(basename "${SHA_FILE}")") + +if [[ "${DRY_RUN}" -eq 1 ]]; then + shasum -a 256 -c "${SHA_FILE}" + echo "Deep Search dry-run artifact verified: ${ARCHIVE}" + exit 0 +fi + +if [[ "${AUTO_YES}" -ne 1 ]]; then + echo "Target: ${HOST}:${RUNTIME_BASE}" + echo "Release: ${RELEASE_ID}" + read -r -p "Install standalone Deep Search on 103? [y/N] " confirm /dev/null || true)" +database_path="${RUNTIME_BASE}/data/research.sqlite" + +mkdir -p "${release_dir}" "${backup_dir}" +cp "${plist}" "${backup_dir}/launchagent.plist" 2>/dev/null || true +if [[ -f "${database_path}" ]]; then + sqlite3 "${database_path}" ".timeout 10000" \ + ".backup '${backup_dir}/research.sqlite'" >/dev/null +fi +cd "${INCOMING}" +shasum -a 256 -c "$(basename "${sha_file}")" +tar -xzf "${archive}" -C "${release_dir}" + +rollback() { + if [[ -n "${previous_target}" && -d "${previous_target}" ]]; then + ln -sfn "${previous_target}" "${current_link}" + else + rm -f "${current_link}" + fi + if [[ -f "${backup_dir}/launchagent.plist" ]]; then + cp "${backup_dir}/launchagent.plist" "${plist}" + launchctl bootout "${gui}/${label}" >/dev/null 2>&1 || true + launchctl bootstrap "${gui}" "${plist}" >/dev/null 2>&1 || true + launchctl kickstart -k "${gui}/${label}" >/dev/null 2>&1 || true + fi +} +trap rollback ERR + +DEEP_SEARCH_RUNTIME_BASE="${RUNTIME_BASE}" \ +DEEP_SEARCH_RELEASE_DIR="${release_dir}" \ + bash "${release_dir}/scripts/install-deep-search-runtime-prod.sh" + +set -a +# shellcheck disable=SC1090 +source "${RUNTIME_BASE}/shared/.env" +set +a + +curl -fsS --max-time 5 "http://127.0.0.1:20100/health" >/dev/null +curl -fsS --max-time 20 \ + -H "X-Secret-Key: ${TKMIND_DEEP_SEARCH_SECRET}" \ + -H "Content-Type: application/json" \ + -d '{"query":"OpenAI","limit":3}' \ + "http://127.0.0.1:20100/v1/search" \ + | grep -q '"results"' + +docker_bin="" +for candidate in /usr/local/bin/docker /opt/homebrew/bin/docker; do + if [[ -x "${candidate}" ]]; then + docker_bin="${candidate}" + break + fi +done +if [[ -n "${docker_bin}" ]]; then + goosed_container="$("${docker_bin}" ps --format '{{.Names}}' | grep -E '^goosed-prod-' | head -1)" + if [[ -n "${goosed_container}" ]]; then + "${docker_bin}" exec "${goosed_container}" sh -lc \ + 'curl -fsS --max-time 5 http://host.docker.internal:20100/health >/dev/null' + fi +fi + +trap - ERR +printf '%s\n' "${RELEASE_ID}" > "${RUNTIME_BASE}/current-release.txt" +REMOTE + +echo "Deep Search installed and verified on 103: ${RELEASE_ID}" diff --git a/scripts/run-deep-search-prod.sh b/scripts/run-deep-search-prod.sh new file mode 100644 index 0000000..842538e --- /dev/null +++ b/scripts/run-deep-search-prod.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNTIME_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}" +ENV_FILE="${DEEP_SEARCH_ENV_FILE:-${RUNTIME_BASE}/shared/.env}" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "[deep-search-start] missing environment file: ${ENV_FILE}" >&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1090 +source "${ENV_FILE}" +set +a + +: "${TKMIND_DEEP_SEARCH_SECRET:?missing TKMIND_DEEP_SEARCH_SECRET}" + +export NODE_ENV=production +export TKMIND_DEEP_SEARCH_HOST="${TKMIND_DEEP_SEARCH_HOST:-127.0.0.1}" +export TKMIND_DEEP_SEARCH_PORT="${TKMIND_DEEP_SEARCH_PORT:-20100}" +export TKMIND_DEEP_SEARCH_DB="${TKMIND_DEEP_SEARCH_DB:-${RUNTIME_BASE}/data/research.sqlite}" +export TKMIND_DEEP_SEARCH_SEARXNG_URL="${TKMIND_DEEP_SEARCH_SEARXNG_URL:-http://127.0.0.1:20080/search}" +export TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL:-http://127.0.0.1:8081/api/internal/deep-search/llm}" +export TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET:-${TKMIND_DEEP_SEARCH_SECRET}}" + +mkdir -p "$(dirname "${TKMIND_DEEP_SEARCH_DB}")" "${RUNTIME_BASE}/logs" + +NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}" +if [[ ! -x "${NODE_BIN}" ]]; then + NODE_BIN="$(command -v node)" +fi + +exec "${NODE_BIN}" "${RUNTIME_ROOT}/deep-search-server.mjs" diff --git a/server.mjs b/server.mjs index 489224e..83b32d2 100644 --- a/server.mjs +++ b/server.mjs @@ -198,6 +198,7 @@ import { createScheduleService } from './schedule-service.mjs'; import { createFeedbackService } from './user-feedback.mjs'; import { startScheduleReminderWorker } from './schedule-reminder-worker.mjs'; import { createLlmProviderService, RELAY_BOOTSTRAP } from './llm-providers.mjs'; +import { executeDeepSearchLlmGateway } from './deep-search-llm-gateway.mjs'; import { createDirectChatService, isDirectChatSessionId, isPortalDirectChatSnapshot, sendDirectChatSessionEvents, shouldExpirePortalDirectChatSnapshot } from './direct-chat-service.mjs'; import { filterUserVisibleConversation, @@ -254,6 +255,7 @@ const API_TARGETS = parseApiTargets(); const API_TARGET = API_TARGETS[0] ?? 'https://127.0.0.1:18006'; const API_SECRET = process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret'; const INTERNAL_AGENT_SECRET = process.env.MINDSPACE_INTERNAL_AGENT_SECRET ?? API_SECRET; +const DEEP_SEARCH_INTERNAL_SECRET = process.env.TKMIND_DEEP_SEARCH_SECRET ?? INTERNAL_AGENT_SECRET; const mindSpaceServerRuntime = resolveMindSpaceServerRuntimeOptions(__dirname, process.env); const ACCESS_PASSWORD = process.env.H5_ACCESS_PASSWORD; const WECHAT_MP_CONFIG = loadWechatMpConfig(); @@ -2127,6 +2129,7 @@ api.use(async (req, res, next) => { await userAuthReady; if (req.path === '/status' || req.path === '/runtime/status') return next(); if (req.path.startsWith('/internal/agent/')) return next(); + if (req.path === '/internal/deep-search/llm') return next(); if (req.path === '/agent/mindspace_page_patch') return next(); if (req.path === '/agent/mindspace_asset_delete') return next(); if (req.path === '/agent/mindspace_asset_download') return next(); @@ -2741,6 +2744,24 @@ function requireInternalAgentSecret(req, res) { return false; } +api.post('/internal/deep-search/llm', async (req, res) => { + try { + const result = await executeDeepSearchLlmGateway({ + llmProviderService, + expectedSecret: DEEP_SEARCH_INTERNAL_SECRET, + providedSecret: bearerToken(req) ?? req.get('x-secret-key'), + input: req.body ?? {}, + }); + return res.status(result.status).json(result.body); + } catch (error) { + console.warn('[deep-search] LLM gateway failed:', error); + return res.status(502).json({ + ok: false, + message: error instanceof Error ? error.message : 'Deep Search LLM gateway failed', + }); + } +}); + api.post('/mindspace/v1/agent/jobs', async (req, res) => { if (!mindSpaceAgentJobs || !ensureMindSpaceEnabled(res, req, { agent: true })) return; try { diff --git a/tkmind-search-mcp.mjs b/tkmind-search-mcp.mjs index 4c01b16..52af9ad 100644 --- a/tkmind-search-mcp.mjs +++ b/tkmind-search-mcp.mjs @@ -1,7 +1,21 @@ import readline from 'node:readline'; -import { normalizeMindSearchConfig } from './mindsearch-config.mjs'; +import { normalizeMindSearchConfig, resolveMindSearchService } from './mindsearch-config.mjs'; import { SEARCH_ERROR_CODES, validateSearchRequest } from './search-capability.mjs'; -import { readSafeUrl, searchGithubCode, searchSearxng } from './mindsearch-providers.mjs'; +import { + cancelResearchTask, + getResearchTask, + readSafeUrl, + searchGithubCode, + searchGithubGateway, + searchResearchService, + searchSearxng, + startResearchTask, +} from './mindsearch-providers.mjs'; + +function parseJson(value, fallback) { + if (!value) return fallback; + try { return JSON.parse(value); } catch { return fallback; } +} const config = normalizeMindSearchConfig({ enabled: process.env.TKMIND_SEARCH_ENABLED, @@ -11,12 +25,51 @@ const config = normalizeMindSearchConfig({ github: process.env.TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED, reader: process.env.TKMIND_SEARCH_READER_ENABLED, }, + settings: { + searxngEndpoint: process.env.TKMIND_SEARCH_SEARXNG_URL, + maxResults: process.env.TKMIND_SEARCH_MAX_RESULTS, + timeoutMs: process.env.TKMIND_SEARCH_TIMEOUT_MS, + readerMaxChars: process.env.TKMIND_SEARCH_READER_MAX_CHARS, + }, + services: parseJson(process.env.TKMIND_SEARCH_SERVICES_JSON, undefined), + routes: parseJson(process.env.TKMIND_SEARCH_ROUTES_JSON, undefined), }); const tools = [ { name: 'tkmind_search', description: 'Optional external search enhancement; existing web search remains primary.', inputSchema: { type: 'object', properties: { query: { type: 'string' }, type: { type: 'string' }, limit: { type: 'number' } }, required: ['query'] } }, { name: 'tkmind_read', description: 'Optional safe URL reader for search citations.', inputSchema: { type: 'object', properties: { url: { type: 'string' } }, required: ['url'] } }, ]; +if (resolveMindSearchService(config, 'research')) { + tools.push({ + name: 'tkmind_research', + description: 'Start an asynchronous deep-research task and return its task identifier.', + inputSchema: { + type: 'object', + properties: { question: { type: 'string' }, depth: { type: 'string', enum: ['quick', 'standard', 'deep'] } }, + required: ['question'], + }, + }); + tools.push( + { + name: 'tkmind_research_status', + description: 'Get Deep Search progress, evidence sources, and the completed report.', + inputSchema: { + type: 'object', + properties: { task_id: { type: 'string' } }, + required: ['task_id'], + }, + }, + { + name: 'tkmind_research_cancel', + description: 'Cancel a queued or running Deep Search task.', + inputSchema: { + type: 'object', + properties: { task_id: { type: 'string' } }, + required: ['task_id'], + }, + }, + ); +} function response(id, result) { process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', id, result })}\n`); } function error(id, code, message) { response(id, { isError: true, content: [{ type: 'text', text: `${code}: ${message}` }], structuredContent: { code, message } }); } @@ -31,15 +84,60 @@ function handle(message) { if (name === 'tkmind_search') { const checked = validateSearchRequest(params.arguments ?? {}); if (!checked.ok) return error(id, checked.code, checked.message); - if (checked.value.type === 'code' && config.providers.github) { - return searchGithubCode(checked.value.query, { limit: checked.value.limit }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: 'github' }) }], structuredContent: { results, query: checked.value.query, provider: 'github' } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + const route = checked.value.type === 'code' ? 'code' : checked.value.type === 'news' ? 'news' : 'web'; + const service = resolveMindSearchService(config, route); + if (service?.adapter === 'github') { + const operation = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL + ? searchGithubGateway + : searchGithubCode; + return operation(checked.value.query, { + limit: checked.value.limit, + endpoint: process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL || service.endpoint || undefined, + }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); } - if (config.providers.searxng) { - return searchSearxng(checked.value.query, { limit: checked.value.limit }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: 'searxng' }) }], structuredContent: { results, query: checked.value.query, provider: 'searxng' } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + if (service?.adapter === 'searxng') { + return searchSearxng(checked.value.query, { limit: checked.value.limit, endpoint: service.endpoint }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + } + if (service?.adapter === 'research-http') { + return searchResearchService(checked.value.query, { endpoint: service.endpoint, type: checked.value.type, limit: checked.value.limit, timeoutMs: service.timeoutMs }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); } return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No search provider is configured.'); } - if (name === 'tkmind_read' && config.providers.reader) return readSafeUrl(params.arguments?.url).then((result) => response(id, { content: [{ type: 'text', text: result.content }], structuredContent: result })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + if (name === 'tkmind_read' && resolveMindSearchService(config, 'read')?.adapter === 'reader') return readSafeUrl(params.arguments?.url).then((result) => response(id, { content: [{ type: 'text', text: result.content }], structuredContent: result })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + if (name === 'tkmind_research') { + const question = String(params.arguments?.question ?? '').trim(); + if (!question || question.length > 2000) return error(id, SEARCH_ERROR_CODES.INVALID_REQUEST, 'question must be 1-2000 characters'); + const depth = ['quick', 'standard', 'deep'].includes(params.arguments?.depth) ? params.arguments.depth : 'standard'; + const service = resolveMindSearchService(config, 'research'); + if (service?.adapter !== 'research-http') return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No research service is configured.'); + return startResearchTask(question, { + endpoint: service.endpoint, + depth, + userId: process.env.TKMIND_SEARCH_USER_ID || null, + llmProviderKeyId: service.llmProviderKeyId || '', + llmModel: service.llmModel || '', + timeoutMs: service.timeoutMs, + }) + .then((result) => response(id, { content: [{ type: 'text', text: JSON.stringify(result) }], structuredContent: result })) + .catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + } + if (name === 'tkmind_research_status' || name === 'tkmind_research_cancel') { + const taskId = String(params.arguments?.task_id ?? '').trim(); + if (!taskId || taskId.length > 200) return error(id, SEARCH_ERROR_CODES.INVALID_REQUEST, 'task_id must be 1-200 characters'); + const service = resolveMindSearchService(config, 'research'); + if (service?.adapter !== 'research-http') return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No research service is configured.'); + const operation = name === 'tkmind_research_cancel' ? cancelResearchTask : getResearchTask; + return operation(taskId, { + endpoint: service.endpoint, + userId: process.env.TKMIND_SEARCH_USER_ID || null, + timeoutMs: service.timeoutMs, + }) + .then((result) => response(id, { + content: [{ type: 'text', text: result.report || JSON.stringify(result) }], + structuredContent: result, + })) + .catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + } return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No reader provider is configured.'); } diff --git a/user-auth.mjs b/user-auth.mjs index a241815..e5022d2 100644 --- a/user-auth.mjs +++ b/user-auth.mjs @@ -1868,7 +1868,12 @@ export function createUserAuth(pool, options = {}) { await syncUserSkillsForUser(user); if (capabilityState.unrestricted) { return { - ...buildAgentExtensionPolicy(capabilityState.capabilities, { unrestricted: true, toolMode, mindSearchConfig }), + ...buildAgentExtensionPolicy(capabilityState.capabilities, { + unrestricted: true, + toolMode, + mindSearchConfig, + userId: user.id, + }), policies: {}, unrestricted: true, toolMode, @@ -1933,6 +1938,7 @@ export function createUserAuth(pool, options = {}) { sandboxMcp, toolMode, mindSearchConfig, + userId: user.id, }), capabilities: effectiveCapabilities, policies: policyState.policies,