diff --git a/capabilities.mjs b/capabilities.mjs index 42ba737..c7576d6 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -401,7 +401,14 @@ function sandboxMcpEnvs(sandboxMcp, mcpTools) { */ export function buildAgentExtensionPolicy( capabilities, - { unrestricted = false, policies = null, sandboxMcp = null, toolMode = 'chat', mindSearchConfig = null } = {}, + { + unrestricted = false, + policies = null, + sandboxMcp = null, + toolMode = 'chat', + mindSearchConfig = null, + userId = null, + } = {}, ) { if (unrestricted) { return { extensionOverrides: null, enableContextMemory: true, gooseMode: 'auto' }; @@ -518,8 +525,16 @@ export function buildAgentExtensionPolicy( TKMIND_SEARCH_READER_MAX_CHARS: String(mindSearchConfig.settings?.readerMaxChars ?? 12000), TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mindSearchConfig.services ?? []), TKMIND_SEARCH_ROUTES_JSON: JSON.stringify(mindSearchConfig.routes ?? {}), + TKMIND_SEARCH_USER_ID: userId ? String(userId) : '', + TKMIND_DEEP_SEARCH_SECRET: process.env.TKMIND_DEEP_SEARCH_SECRET ?? '', }, - available_tools: ['tkmind_search', 'tkmind_read', ...(hasResearchService ? ['tkmind_research'] : [])], + available_tools: [ + 'tkmind_search', + 'tkmind_read', + ...(hasResearchService + ? ['tkmind_research', 'tkmind_research_status', 'tkmind_research_cancel'] + : []), + ], }); } if (capabilities.excel_analysis) { diff --git a/deep-search-engine.mjs b/deep-search-engine.mjs new file mode 100644 index 0000000..ee58bfe --- /dev/null +++ b/deep-search-engine.mjs @@ -0,0 +1,698 @@ +import dns from 'node:dns/promises'; +import { EventEmitter } from 'node:events'; +import net from 'node:net'; +import { randomUUID } from 'node:crypto'; +import { searchSearxng } from './mindsearch-providers.mjs'; + +const DEPTH_PROFILES = Object.freeze({ + quick: { goals: 3, rounds: 2, resultsPerQuery: 6, maxSources: 8 }, + standard: { goals: 4, rounds: 2, resultsPerQuery: 7, maxSources: 14 }, + deep: { goals: 6, rounds: 3, resultsPerQuery: 10, maxSources: 24 }, +}); + +const TRACKING_PARAMS = /^(utm_|fbclid$|gclid$|mc_)/i; + +function clamp(value, min, max) { + return Math.max(min, Math.min(max, value)); +} + +function throwIfAborted(signal) { + if (signal?.aborted) { + const error = new Error('Research task was cancelled'); + error.name = 'AbortError'; + throw error; + } +} + +function decodeHtml(value) { + return value + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/</gi, '<') + .replace(/>/gi, '>') + .replace(/"/gi, '"') + .replace(/'|'/gi, "'") + .replace(/—/gi, '—') + .replace(/–/gi, '–') + .replace(/…/gi, '…') + .replace(/&#(\d+);/g, (_match, code) => String.fromCodePoint(Number(code))); +} + +export function htmlToPlainText(html) { + return decodeHtml(String(html ?? '') + .replace(/]*>[\s\S]*?<\/script>/gi, ' ') + .replace(/]*>[\s\S]*?<\/style>/gi, ' ') + .replace(/]*>[\s\S]*?<\/noscript>/gi, ' ') + .replace(//g, ' ') + .replace(/<\/(p|div|section|article|li|h[1-6]|tr)>/gi, '\n') + .replace(//gi, '\n') + .replace(/<[^>]+>/g, ' ')) + .replace(/[ \t]+/g, ' ') + .replace(/\n\s+/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); +} + +export function tokenize(value) { + const normalized = String(value ?? '').toLowerCase(); + const latin = normalized.match(/[a-z0-9][a-z0-9._-]{1,}/g) ?? []; + const cjkSequences = normalized.match(/[\p{Script=Han}]{2,}/gu) ?? []; + const cjk = []; + for (const sequence of cjkSequences) { + cjk.push(sequence); + for (let index = 0; index < sequence.length - 1; index += 1) { + cjk.push(sequence.slice(index, index + 2)); + } + } + return [...new Set([...latin, ...cjk])]; +} + +function lexicalScore(needle, haystack) { + const terms = tokenize(needle); + if (!terms.length) return 0; + const text = String(haystack ?? '').toLowerCase(); + return terms.filter((term) => text.includes(term)).length / terms.length; +} + +export function canonicalizeUrl(value) { + try { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol)) return ''; + url.hash = ''; + for (const key of [...url.searchParams.keys()]) { + if (TRACKING_PARAMS.test(key)) url.searchParams.delete(key); + } + url.hostname = url.hostname.toLowerCase(); + if ((url.protocol === 'https:' && url.port === '443') || (url.protocol === 'http:' && url.port === '80')) { + url.port = ''; + } + return url.toString(); + } catch { + return ''; + } +} + +function isPrivateIp(address) { + const normalized = String(address ?? '').toLowerCase(); + if (net.isIPv4(normalized)) { + const parts = normalized.split('.').map(Number); + return parts[0] === 0 + || parts[0] === 10 + || parts[0] === 127 + || (parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) + || (parts[0] === 169 && parts[1] === 254) + || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) + || (parts[0] === 192 && parts[1] === 168) + || parts[0] >= 224; + } + if (net.isIPv6(normalized)) { + return normalized === '::' + || normalized === '::1' + || normalized.startsWith('fc') + || normalized.startsWith('fd') + || normalized.startsWith('fe8') + || normalized.startsWith('fe9') + || normalized.startsWith('fea') + || normalized.startsWith('feb') + || normalized.startsWith('::ffff:127.') + || normalized.startsWith('::ffff:10.') + || normalized.startsWith('::ffff:192.168.'); + } + return false; +} + +export async function assertSafePublicUrl(value, { lookup = dns.lookup } = {}) { + const url = new URL(value); + if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password) { + throw new Error('Unsafe source URL'); + } + if (url.hostname === 'localhost' || isPrivateIp(url.hostname)) throw new Error('Unsafe source URL'); + const records = await lookup(url.hostname, { all: true, verbatim: true }); + if (!records.length || records.some((record) => isPrivateIp(record.address))) { + throw new Error('Unsafe source URL'); + } + return url; +} + +export async function readResearchSource(target, { + fetchImpl = fetch, + lookup = dns.lookup, + timeoutMs = 10_000, + maxChars = 40_000, + signal, + redirects = 3, +} = {}) { + let current = String(target); + for (let attempt = 0; attempt <= redirects; attempt += 1) { + throwIfAborted(signal); + const url = await assertSafePublicUrl(current, { lookup }); + const timeout = AbortSignal.timeout(timeoutMs); + const combinedSignal = signal ? AbortSignal.any([signal, timeout]) : timeout; + const response = await fetchImpl(url, { + headers: { + accept: 'text/html,text/plain,application/xhtml+xml', + 'user-agent': 'TKMind-Deep-Search/1.0', + }, + redirect: 'manual', + signal: combinedSignal, + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get('location'); + if (!location || attempt === redirects) throw new Error('Unsafe or excessive redirect'); + current = new URL(location, url).toString(); + continue; + } + if (!response.ok) throw new Error(`Reader returned ${response.status}`); + const contentType = response.headers.get('content-type') ?? ''; + if (!/text\/|application\/xhtml\+xml/i.test(contentType)) { + throw new Error(`Unsupported content type: ${contentType || 'unknown'}`); + } + const raw = (await response.text()).slice(0, maxChars * 4); + return { + url: url.toString(), + title: decodeHtml(raw.match(/]*>([\s\S]*?)<\/title>/i)?.[1] ?? '').trim().slice(0, 500), + content: (/html|xhtml/i.test(contentType) ? htmlToPlainText(raw) : raw).slice(0, maxChars), + contentType, + }; + } + throw new Error('Reader failed'); +} + +function fallbackGoals(question) { + const latinEntity = (String(question).match(/[A-Za-z][A-Za-z0-9._-]{2,}/g) ?? [])[0] ?? ''; + const candidates = []; + const add = (pattern, goal, suffix, english) => { + if (pattern.test(question)) { + candidates.push({ + goal, + query: `${question} ${suffix}`, + alternate: latinEntity ? `${latinEntity} ${english}` : `${question} ${suffix} 权威来源`, + }); + } + }; + add(/架构|组成|原理|architecture/i, '技术架构与组成', '技术架构 组件 原理', 'architecture components official documentation'); + add(/特性|功能|能力|feature|capabilit/i, '核心特性与能力', '核心特性 功能', 'features capabilities official documentation'); + add(/局限|风险|问题|争议|limitation|risk|issue/i, '主要局限、风险与争议', '局限 风险 已知问题', 'limitations known issues privacy'); + add(/市场|规模|商业|竞争|market|business/i, '市场规模与竞争格局', '市场规模 竞争格局 数据', 'market size competitors report'); + add(/对比|比较|区别|versus|compare/i, '替代方案与横向比较', '替代方案 对比', 'alternatives comparison'); + add(/趋势|未来|战略|影响|trend|future|strategy/i, '趋势、影响与未来判断', '趋势 影响 未来', 'trends roadmap future'); + const generic = [ + { goal: '核心事实与定义', query: question, alternate: latinEntity ? `${latinEntity} official documentation overview` : `${question} 权威概述` }, + { goal: '最新进展与时间线', query: `${question} 最新进展 时间线`, alternate: latinEntity ? `${latinEntity} releases changelog latest` : `${question} 最新时间线` }, + { goal: '权威及一手来源', query: `${question} 官方 报告 数据`, alternate: latinEntity ? `${latinEntity} official documentation GitHub` : `${question} 一手来源` }, + { goal: '主要参与者与方案比较', query: `${question} 主要参与者 对比`, alternate: `${question} alternatives comparison` }, + { goal: '风险、争议与反方观点', query: `${question} 风险 争议 局限`, alternate: `${question} limitations criticism` }, + { goal: '趋势、影响与未来判断', query: `${question} 趋势 影响 未来`, alternate: `${question} roadmap future` }, + ]; + const seen = new Set(candidates.map((item) => item.goal)); + return [...candidates, ...generic.filter((item) => !seen.has(item.goal))]; +} + +export function buildFallbackResearchPlan(question, depth = 'standard') { + const profile = DEPTH_PROFILES[depth] ?? DEPTH_PROFILES.standard; + return fallbackGoals(question).slice(0, profile.goals).map((item, index) => ({ + id: `goal-${index + 1}`, + goal: item.goal, + queries: [item.query, item.alternate], + sources: index === 2 ? ['web', 'paper'] : ['web'], + })); +} + +function normalizePlan(candidate, question, depth) { + const profile = DEPTH_PROFILES[depth] ?? DEPTH_PROFILES.standard; + const source = Array.isArray(candidate) ? candidate : candidate?.research_plan; + if (!Array.isArray(source) || !source.length) return buildFallbackResearchPlan(question, depth); + const plan = source.map((step, index) => ({ + id: `goal-${index + 1}`, + goal: String(step.goal ?? '').trim().slice(0, 300), + queries: (Array.isArray(step.queries) ? step.queries : step.query) + ?.map?.((query) => String(query ?? '').trim().slice(0, 500)) + .filter(Boolean) + .slice(0, 4) ?? [], + sources: (Array.isArray(step.sources) ? step.sources : ['web']) + .map((source) => String(source)) + .filter((source) => ['web', 'news', 'paper', 'code'].includes(source)) + .slice(0, 4), + })).filter((step) => step.goal && step.queries.length).slice(0, profile.goals); + return plan.length ? plan : buildFallbackResearchPlan(question, depth); +} + +export function createOpenAiCompatibleResearchLlm({ + endpoint = process.env.TKMIND_DEEP_SEARCH_LLM_URL, + apiKey = process.env.TKMIND_DEEP_SEARCH_LLM_API_KEY, + model = process.env.TKMIND_DEEP_SEARCH_LLM_MODEL, + fetchImpl = fetch, + timeoutMs = 90_000, +} = {}) { + if (!endpoint || !model) return null; + async function complete(messages, { json = false } = {}) { + const response = await fetchImpl(endpoint, { + method: 'POST', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + 'content-type': 'application/json', + ...(apiKey ? { authorization: `Bearer ${apiKey}` } : {}), + }, + body: JSON.stringify({ + model, + messages, + temperature: 0.2, + ...(json ? { response_format: { type: 'json_object' } } : {}), + }), + }); + if (!response.ok) throw new Error(`Research LLM returned ${response.status}`); + const body = await response.json(); + return String(body.choices?.[0]?.message?.content ?? '').trim(); + } + return { + async plan(question, depth) { + const content = await complete([ + { + role: 'system', + content: 'You are a research planner. Return JSON {"research_plan":[{"goal":"","queries":[""],"sources":["web"]}]}. Create distinct, verifiable goals and search queries.', + }, + { role: 'user', content: `Depth: ${depth}\nQuestion: ${question}` }, + ], { json: true }); + return JSON.parse(content); + }, + async synthesize({ question, plan, evidence }) { + return complete([ + { + role: 'system', + content: 'Write a rigorous Markdown research report using only the supplied evidence. Cite every factual claim with [n]. Include executive summary, findings by research goal, uncertainties, and sources. Never invent citations.', + }, + { + role: 'user', + content: JSON.stringify({ question, plan, evidence }, null, 2).slice(0, 120_000), + }, + ]); + }, + }; +} + +function sourceAuthority(url) { + try { + const hostname = new URL(url).hostname; + if (/\.(gov|edu)$/.test(hostname) || hostname === 'arxiv.org') return 0.2; + if (hostname === 'github.com' || hostname.endsWith('.org')) return 0.12; + } catch { + return 0; + } + return 0; +} + +function scoreSource(source, question, goal = '') { + const haystack = `${source.title} ${source.snippet} ${source.content}`; + const base = clamp( + lexicalScore(question, haystack) * 0.45 + + lexicalScore(goal, haystack) * 0.35 + + sourceAuthority(source.url) + + (/official|documentation|docs|官方|文档/i.test(source.title) ? 0.08 : 0) + + (source.content?.length > 1000 ? 0.05 : 0) + + (source.rank ? 0.05 / source.rank : 0), + 0, + 1, + ); + const entity = (String(question).match(/[A-Za-z][A-Za-z0-9._-]{2,}/g) ?? [])[0]; + return entity && !haystack.toLowerCase().includes(entity.toLowerCase()) ? base * 0.25 : base; +} + +function selectSourcesForReading(sourceMap, plan, maxSources) { + const all = [...sourceMap.values()].sort((a, b) => b.score - a.score); + const selected = []; + const seen = new Set(); + for (const step of plan) { + for (const source of all.filter((item) => item.goal === step.goal).slice(0, 2)) { + if (seen.has(source.url)) continue; + seen.add(source.url); + selected.push(source); + } + } + for (const source of all) { + if (selected.length >= maxSources) break; + if (seen.has(source.url)) continue; + seen.add(source.url); + selected.push(source); + } + return selected.slice(0, maxSources); +} + +function splitSentences(content) { + return String(content ?? '') + .split(/(?<=[。！？.!?])\s+|\n+/u) + .map((sentence) => sentence.trim()) + .filter((sentence) => sentence.length >= 35 && sentence.length <= 1200); +} + +export function extractEvidence(source, { question, goal }) { + const normalizedTitle = String(source.title ?? '').toLowerCase().replace(/\s+/g, ' ').trim(); + const sentences = splitSentences(source.content || source.snippet) + .filter((claim) => { + const normalized = claim.toLowerCase().replace(/\s+/g, ' ').trim(); + if (!normalized || normalized === normalizedTitle) return false; + if (normalized.startsWith(normalizedTitle) && normalized.length < normalizedTitle.length + 30) return false; + return !/contents\s+menu\s+expand|light mode\s+dark mode|sign in\s+sign up|data-hydro-click|class="|aria-[a-z-]+=/i.test(normalized); + }); + const ranked = sentences + .map((claim) => ({ + claim, + score: lexicalScore(`${question} ${goal}`, claim) + Math.min(0.25, claim.length / 1000), + })) + .sort((a, b) => b.score - a.score) + .slice(0, 2); + if (!ranked.length && source.snippet) ranked.push({ claim: source.snippet, score: 0.2 }); + return ranked.map((item) => ({ + claim: item.claim, + confidence: clamp(0.45 + item.score * 0.45 + sourceAuthority(source.url), 0.45, 0.95), + url: source.url, + title: source.title, + goal, + })); +} + +async function runBounded(items, limit, worker) { + const output = new Array(items.length); + let cursor = 0; + async function run() { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + output[index] = await worker(items[index], index); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run)); + return output; +} + +function reportFromEvidence({ question, plan, evidence, sources }) { + const sourceIndex = new Map(sources.map((source, index) => [source.url, index + 1])); + const lines = [ + `# ${question}`, + '', + '## 执行摘要', + '', + ]; + const strongest = [...evidence].sort((a, b) => b.confidence - a.confidence).slice(0, 5); + if (!strongest.length) { + lines.push('当前检索未获得足够的可验证证据，建议调整问题或检查搜索服务。'); + } else { + for (const item of strongest) lines.push(`- ${item.claim} [${sourceIndex.get(item.url)}]`); + } + for (const step of plan) { + lines.push('', `## ${step.goal}`, ''); + const items = evidence.filter((item) => item.goal === step.goal).slice(0, 5); + if (!items.length) lines.push('该研究目标暂未获得足够证据。'); + for (const item of items) { + lines.push(`- ${item.claim} [${sourceIndex.get(item.url)}]`); + } + } + lines.push('', '## 不确定性与局限', ''); + lines.push('- 报告仅基于本次检索可访问的公开网页；付费墙、动态页面和非文本资料可能未被读取。'); + lines.push('- 自动提取的事实仍建议对照一手来源复核，尤其是数字、日期和预测。'); + lines.push('', '## 来源', ''); + sources.forEach((source, index) => { + lines.push(`${index + 1}. [${source.title || source.url}](${source.url}) — ${source.provider}`); + }); + return lines.join('\n'); +} + +function citationNumbers(markdown) { + return [...String(markdown).matchAll(/\[(\d+)\]/g)].map((match) => Number(match[1])); +} + +function validLlmReport(markdown, sourceCount) { + const citations = citationNumbers(markdown); + return markdown.length >= 200 + && citations.length > 0 + && citations.every((number) => number >= 1 && number <= sourceCount); +} + +export function createSearxngResearchProvider({ + endpoint = process.env.TKMIND_DEEP_SEARCH_SEARXNG_URL || process.env.TKMIND_SEARCH_SEARXNG_URL || 'http://127.0.0.1:8080/search', + fetchImpl = fetch, +} = {}) { + return { + name: 'searxng', + async search(query, { limit, signal } = {}) { + throwIfAborted(signal); + return searchSearxng(query, { limit, endpoint, fetchImpl, signal }); + }, + }; +} + +export function createDeepSearchEngine({ + store, + searchProvider = createSearxngResearchProvider(), + reader = readResearchSource, + llm = createOpenAiCompatibleResearchLlm(), + memorySink = null, + idFactory = () => randomUUID(), + now = () => Date.now(), + concurrency = 3, +} = {}) { + if (!store) throw new Error('Deep Search store is required'); + const running = new Map(); + const emitter = new EventEmitter(); + + function publish(taskId, type, payload = {}) { + store.appendEvent(taskId, type, payload); + emitter.emit(taskId, { type, payload, createdAt: now() }); + } + + async function planResearch(question, depth) { + if (llm?.plan) { + try { + return normalizePlan(await llm.plan(question, depth), question, depth); + } catch { + // The deterministic planner keeps Deep Search available when the LLM is unavailable. + } + } + return buildFallbackResearchPlan(question, depth); + } + + async function executeTask(taskId, signal) { + const task = store.getTask(taskId, { includeSources: false, includeEvents: false }); + const profile = DEPTH_PROFILES[task.depth] ?? DEPTH_PROFILES.standard; + try { + store.updateTask(taskId, { status: 'researching', phase: 'planning', progress: 5 }); + publish(taskId, 'phase', { phase: 'planning', progress: 5 }); + const plan = await planResearch(task.question, task.depth); + throwIfAborted(signal); + store.updateTask(taskId, { plan, phase: 'searching', progress: 12 }); + publish(taskId, 'plan', { plan }); + + const sourcesByUrl = new Map(); + for (let round = 0; round < profile.rounds; round += 1) { + throwIfAborted(signal); + const jobs = plan.map((step) => ({ + step, + query: step.queries[round % step.queries.length], + round, + })); + const batches = await runBounded(jobs, concurrency, async ({ step, query }) => { + try { + const results = await searchProvider.search(query, { + limit: profile.resultsPerQuery, + source: step.sources[0], + signal, + }); + publish(taskId, 'search', { goal: step.goal, query, count: results.length, round: round + 1 }); + return results.map((result) => ({ ...result, goal: step.goal, query })); + } catch (error) { + if (error?.name === 'AbortError') throw error; + publish(taskId, 'search_error', { goal: step.goal, query, message: error.message }); + return []; + } + }); + for (const source of batches.flat()) { + const url = canonicalizeUrl(source.url); + if (!url) continue; + const candidate = { + ...source, + url, + provider: source.provider || source.source || searchProvider.name || 'unknown', + score: scoreSource(source, task.question, source.goal), + }; + const previous = sourcesByUrl.get(url); + if (!previous || candidate.score > previous.score) sourcesByUrl.set(url, candidate); + } + const progress = 12 + Math.round(((round + 1) / profile.rounds) * 28); + store.updateTask(taskId, { progress, phase: 'searching' }); + publish(taskId, 'progress', { phase: 'searching', progress, sources: sourcesByUrl.size }); + } + + const ranked = selectSourcesForReading(sourcesByUrl, plan, profile.maxSources); + store.updateTask(taskId, { phase: 'reading', progress: 45 }); + publish(taskId, 'phase', { phase: 'reading', progress: 45, sources: ranked.length }); + const readSources = await runBounded(ranked, concurrency, async (source, index) => { + throwIfAborted(signal); + let enriched = source; + try { + const document = await reader(source.url, { signal }); + enriched = { + ...source, + title: document.title || source.title, + content: document.content, + }; + } catch (error) { + if (error?.name === 'AbortError') throw error; + publish(taskId, 'read_error', { url: source.url, message: error.message }); + } + enriched.score = scoreSource(enriched, task.question, enriched.goal); + store.addSource(taskId, enriched); + const progress = 45 + Math.round(((index + 1) / Math.max(1, ranked.length)) * 25); + publish(taskId, 'read', { url: source.url, progress }); + return enriched; + }); + + throwIfAborted(signal); + const finalSources = readSources.sort((a, b) => b.score - a.score); + const evidence = []; + const seenClaims = new Set(); + for (const source of finalSources) { + for (const item of extractEvidence(source, { question: task.question, goal: source.goal })) { + const key = item.claim.toLowerCase().replace(/\W+/gu, '').slice(0, 180); + if (!key || seenClaims.has(key)) continue; + seenClaims.add(key); + evidence.push(item); + } + } + store.updateTask(taskId, { phase: 'synthesizing', progress: 78 }); + publish(taskId, 'phase', { phase: 'synthesizing', progress: 78, evidence: evidence.length }); + let report = ''; + if (llm?.synthesize && finalSources.length) { + try { + const numberedEvidence = evidence.map((item) => ({ + ...item, + citation: finalSources.findIndex((source) => source.url === item.url) + 1, + })); + const candidate = await llm.synthesize({ question: task.question, plan, evidence: numberedEvidence }); + if (validLlmReport(candidate, finalSources.length)) report = candidate; + } catch { + // Fall through to the citation-safe deterministic report. + } + } + if (!report) report = reportFromEvidence({ question: task.question, plan, evidence, sources: finalSources }); + throwIfAborted(signal); + const completedAt = now(); + store.updateTask(taskId, { + status: 'completed', + phase: 'completed', + progress: 100, + report, + completedAt, + }); + store.saveMemory({ + id: `research:${taskId}`, + userId: task.userId, + topic: task.question, + summary: report.slice(0, 4000), + sourceTaskId: taskId, + }); + if (typeof memorySink === 'function' && task.userId) { + try { + await memorySink({ + userId: task.userId, + taskId, + topic: task.question, + summary: report.slice(0, 4000), + sources: finalSources.map((source) => ({ + title: source.title, + url: source.url, + provider: source.provider, + })), + }); + publish(taskId, 'memory_saved', { userId: task.userId }); + } catch (error) { + publish(taskId, 'memory_error', { message: String(error?.message ?? error) }); + } + } + publish(taskId, 'completed', { + progress: 100, + sources: finalSources.length, + evidence: evidence.length, + }); + } catch (error) { + if (error?.name === 'AbortError' || signal.aborted) { + store.updateTask(taskId, { + status: 'cancelled', + phase: 'cancelled', + error: null, + completedAt: now(), + }); + publish(taskId, 'cancelled', {}); + } else { + store.updateTask(taskId, { + status: 'failed', + phase: 'failed', + error: String(error?.message ?? error).slice(0, 2000), + completedAt: now(), + }); + publish(taskId, 'failed', { message: String(error?.message ?? error) }); + } + } finally { + running.delete(taskId); + } + } + + return { + start({ question, depth = 'standard', userId = null } = {}) { + const normalizedQuestion = String(question ?? '').trim(); + if (!normalizedQuestion || normalizedQuestion.length > 2000) { + throw new Error('question must be 1-2000 characters'); + } + const normalizedDepth = Object.hasOwn(DEPTH_PROFILES, depth) ? depth : 'standard'; + const taskId = idFactory(); + store.createTask({ id: taskId, userId, question: normalizedQuestion, depth: normalizedDepth }); + const controller = new AbortController(); + running.set(taskId, controller); + queueMicrotask(() => executeTask(taskId, controller.signal)); + return { taskId, status: 'queued', service: 'tkmind-deep-search' }; + }, + + async search({ query, limit = 10, type = 'web' } = {}) { + const normalizedQuery = String(query ?? '').trim(); + if (!normalizedQuery || normalizedQuery.length > 500) throw new Error('query must be 1-500 characters'); + const results = await searchProvider.search(normalizedQuery, { + limit: clamp(Number(limit) || 10, 1, 20), + source: type, + }); + return results; + }, + + getTask(id) { + return store.getTask(id); + }, + + listTasks(limit) { + return store.listTasks(limit); + }, + + cancel(id) { + const task = store.getTask(id, { includeSources: false, includeEvents: false }); + if (!task) return null; + if (['completed', 'failed', 'cancelled'].includes(task.status)) return task; + running.get(String(id))?.abort(); + store.updateTask(id, { status: 'cancelling', phase: 'cancelling' }); + publish(id, 'cancelling', {}); + return store.getTask(id); + }, + + subscribe(id, listener) { + emitter.on(String(id), listener); + return () => emitter.off(String(id), listener); + }, + + getHealth() { + return { + ok: true, + service: 'tkmind-deep-search', + running: running.size, + tasks: store.getStats(), + llmEnabled: Boolean(llm), + provider: searchProvider.name ?? 'custom', + }; + }, + }; +} diff --git a/deep-search-server.mjs b/deep-search-server.mjs new file mode 100644 index 0000000..9565d32 --- /dev/null +++ b/deep-search-server.mjs @@ -0,0 +1,187 @@ +import http from 'node:http'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { createDeepSearchEngine } from './deep-search-engine.mjs'; +import { createDeepSearchStore } from './deep-search-store.mjs'; + +function sendJson(res, status, body) { + const payload = JSON.stringify(body); + res.writeHead(status, { + 'content-type': 'application/json; charset=utf-8', + 'content-length': Buffer.byteLength(payload), + 'cache-control': 'no-store', + }); + res.end(payload); +} + +async function readJson(req, { maxBytes = 1_000_000 } = {}) { + const chunks = []; + let size = 0; + for await (const chunk of req) { + size += chunk.length; + if (size > maxBytes) { + const error = new Error('Request body is too large'); + error.status = 413; + throw error; + } + chunks.push(chunk); + } + if (!chunks.length) return {}; + try { + return JSON.parse(Buffer.concat(chunks).toString('utf8')); + } catch { + const error = new Error('Request body must be valid JSON'); + error.status = 400; + throw error; + } +} + +function publicTask(task, { includeContent = false, includeEvents = false } = {}) { + if (!task) return null; + return { + ...task, + sources: task.sources?.map((source) => ({ + ...source, + ...(!includeContent ? { content: undefined } : {}), + })), + ...(!includeEvents ? { events: undefined } : {}), + }; +} + +function matchTaskPath(pathname, suffix = '') { + const match = pathname.match(new RegExp(`^/v1/research/([^/]+)${suffix}$`)); + return match ? decodeURIComponent(match[1]) : null; +} + +function canAccessTask(task, requestUserId) { + return !requestUserId || !task?.userId || task.userId === requestUserId; +} + +export function createDeepSearchHttpServer({ + engine, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET || '', + logger = console, +} = {}) { + if (!engine) throw new Error('Deep Search engine is required'); + return http.createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://deep-search.local'); + if (url.pathname === '/health' && req.method === 'GET') { + return sendJson(res, 200, engine.getHealth()); + } + if (secret && req.headers['x-secret-key'] !== secret) { + return sendJson(res, 401, { code: 'UNAUTHORIZED', message: 'Invalid service secret' }); + } + const requestUserId = String(req.headers['x-user-id'] ?? '').trim(); + try { + if (url.pathname === '/v1/search' && req.method === 'POST') { + const input = await readJson(req); + const results = await engine.search(input); + return sendJson(res, 200, { query: input.query, results, service: 'tkmind-deep-search' }); + } + if (url.pathname === '/v1/research' && req.method === 'POST') { + const input = await readJson(req); + const task = engine.start({ + ...input, + userId: requestUserId || input.userId || null, + }); + return sendJson(res, 202, { + task_id: task.taskId, + status: task.status, + service: task.service, + }); + } + if (url.pathname === '/v1/research' && req.method === 'GET') { + const tasks = engine.listTasks(url.searchParams.get('limit')) + .filter((task) => canAccessTask(task, requestUserId)); + return sendJson(res, 200, { tasks }); + } + const eventTaskId = matchTaskPath(url.pathname, '/events'); + if (eventTaskId && req.method === 'GET') { + const task = engine.getTask(eventTaskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + if (!canAccessTask(task, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + res.writeHead(200, { + 'content-type': 'text/event-stream', + 'cache-control': 'no-cache, no-transform', + connection: 'keep-alive', + }); + for (const event of task.events ?? []) { + res.write(`id: ${event.seq}\nevent: ${event.type}\ndata: ${JSON.stringify(event.payload)}\n\n`); + } + const unsubscribe = engine.subscribe(eventTaskId, (event) => { + res.write(`event: ${event.type}\ndata: ${JSON.stringify(event.payload)}\n\n`); + if (['completed', 'failed', 'cancelled'].includes(event.type)) res.end(); + }); + const heartbeat = setInterval(() => res.write(': heartbeat\n\n'), 15_000); + const cleanup = () => { + clearInterval(heartbeat); + unsubscribe(); + }; + req.once('close', cleanup); + res.once('close', cleanup); + return; + } + const taskId = matchTaskPath(url.pathname); + if (taskId && req.method === 'GET') { + const task = engine.getTask(taskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + if (!canAccessTask(task, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + return sendJson(res, 200, publicTask(task, { + includeContent: url.searchParams.get('include_content') === '1', + includeEvents: url.searchParams.get('include_events') === '1', + })); + } + if (taskId && req.method === 'DELETE') { + const existing = engine.getTask(taskId); + if (existing && !canAccessTask(existing, requestUserId)) { + return sendJson(res, 403, { code: 'FORBIDDEN', message: 'Research task belongs to another user' }); + } + const task = engine.cancel(taskId); + if (!task) return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Research task not found' }); + return sendJson(res, 202, publicTask(task)); + } + return sendJson(res, 404, { code: 'NOT_FOUND', message: 'Route not found' }); + } catch (error) { + logger.warn?.('[deep-search] request failed', error); + return sendJson(res, Number(error?.status) || 400, { + code: 'REQUEST_FAILED', + message: String(error?.message ?? error), + }); + } + }); +} + +export function startDeepSearchServer({ + host = process.env.TKMIND_DEEP_SEARCH_HOST || '127.0.0.1', + port = Number(process.env.TKMIND_DEEP_SEARCH_PORT || 20100), + databasePath = process.env.TKMIND_DEEP_SEARCH_DB + || path.join(process.cwd(), '.deep-search', 'research.sqlite'), + logger = console, +} = {}) { + const store = createDeepSearchStore({ databasePath }); + const engine = createDeepSearchEngine({ store }); + const server = createDeepSearchHttpServer({ engine, logger }); + server.listen(port, host, () => { + logger.info?.(`[deep-search] listening on http://${host}:${port}`); + }); + const close = async () => { + await new Promise((resolve) => server.close(resolve)); + store.close(); + }; + return { server, engine, store, close }; +} + +const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ''; +if (import.meta.url === invokedPath) { + const runtime = startDeepSearchServer(); + const shutdown = async () => { + await runtime.close(); + process.exit(0); + }; + process.once('SIGINT', shutdown); + process.once('SIGTERM', shutdown); +} diff --git a/deep-search-store.mjs b/deep-search-store.mjs new file mode 100644 index 0000000..d67c1a9 --- /dev/null +++ b/deep-search-store.mjs @@ -0,0 +1,249 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +const TASK_COLUMNS = new Set([ + 'status', + 'progress', + 'phase', + 'plan_json', + 'report', + 'error', + 'updated_at', + 'completed_at', +]); + +function parseJson(value, fallback) { + if (!value) return fallback; + try { + return JSON.parse(value); + } catch { + return fallback; + } +} + +function mapTask(row) { + if (!row) return null; + return { + id: row.id, + userId: row.user_id || null, + question: row.question, + depth: row.depth, + status: row.status, + progress: row.progress, + phase: row.phase, + plan: parseJson(row.plan_json, []), + report: row.report || '', + error: row.error || null, + createdAt: row.created_at, + updatedAt: row.updated_at, + completedAt: row.completed_at || null, + }; +} + +function mapSource(row) { + return { + id: row.id, + taskId: row.task_id, + url: row.url, + title: row.title, + snippet: row.snippet, + content: row.content, + provider: row.provider, + score: row.score, + query: row.query, + createdAt: row.created_at, + }; +} + +export function createDeepSearchStore({ + databasePath = process.env.TKMIND_DEEP_SEARCH_DB || path.join(process.cwd(), '.deep-search', 'research.sqlite'), + now = () => Date.now(), +} = {}) { + if (databasePath !== ':memory:') { + fs.mkdirSync(path.dirname(path.resolve(databasePath)), { recursive: true }); + } + const db = new DatabaseSync(databasePath); + db.exec(` + PRAGMA journal_mode = WAL; + PRAGMA foreign_keys = ON; + CREATE TABLE IF NOT EXISTS research_tasks ( + id TEXT PRIMARY KEY, + user_id TEXT, + question TEXT NOT NULL, + depth TEXT NOT NULL, + status TEXT NOT NULL, + progress INTEGER NOT NULL DEFAULT 0, + phase TEXT NOT NULL DEFAULT 'queued', + plan_json TEXT NOT NULL DEFAULT '[]', + report TEXT NOT NULL DEFAULT '', + error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + completed_at INTEGER + ); + CREATE TABLE IF NOT EXISTS research_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + url TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + snippet TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT 'unknown', + score REAL NOT NULL DEFAULT 0, + query TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + UNIQUE(task_id, url) + ); + CREATE TABLE IF NOT EXISTS research_events ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + type TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}', + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS research_memories ( + id TEXT PRIMARY KEY, + user_id TEXT, + topic TEXT NOT NULL, + summary TEXT NOT NULL, + source_task_id TEXT NOT NULL REFERENCES research_tasks(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_research_tasks_status ON research_tasks(status, updated_at DESC); + CREATE INDEX IF NOT EXISTS idx_research_sources_task ON research_sources(task_id, score DESC); + CREATE INDEX IF NOT EXISTS idx_research_events_task ON research_events(task_id, seq); + `); + + const createTaskStmt = db.prepare(` + INSERT INTO research_tasks + (id, user_id, question, depth, status, progress, phase, created_at, updated_at) + VALUES (?, ?, ?, ?, 'queued', 0, 'queued', ?, ?) + `); + const getTaskStmt = db.prepare('SELECT * FROM research_tasks WHERE id = ?'); + const listTasksStmt = db.prepare('SELECT * FROM research_tasks ORDER BY created_at DESC LIMIT ?'); + const sourcesStmt = db.prepare('SELECT * FROM research_sources WHERE task_id = ? ORDER BY score DESC, id ASC'); + const eventsStmt = db.prepare('SELECT * FROM research_events WHERE task_id = ? ORDER BY seq ASC'); + const addSourceStmt = db.prepare(` + INSERT INTO research_sources + (task_id, url, title, snippet, content, provider, score, query, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(task_id, url) DO UPDATE SET + title = excluded.title, + snippet = excluded.snippet, + content = CASE + WHEN length(excluded.content) > length(research_sources.content) THEN excluded.content + ELSE research_sources.content + END, + provider = excluded.provider, + score = MAX(research_sources.score, excluded.score), + query = excluded.query + `); + const appendEventStmt = db.prepare(` + INSERT INTO research_events (task_id, type, payload_json, created_at) + VALUES (?, ?, ?, ?) + `); + const saveMemoryStmt = db.prepare(` + INSERT INTO research_memories (id, user_id, topic, summary, source_task_id, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET summary = excluded.summary + `); + + function getTask(id, { includeSources = true, includeEvents = true } = {}) { + const task = mapTask(getTaskStmt.get(String(id))); + if (!task) return null; + if (includeSources) task.sources = sourcesStmt.all(task.id).map(mapSource); + if (includeEvents) { + task.events = eventsStmt.all(task.id).map((event) => ({ + seq: event.seq, + type: event.type, + payload: parseJson(event.payload_json, {}), + createdAt: event.created_at, + })); + } + return task; + } + + return { + createTask({ id, userId = null, question, depth }) { + const timestamp = now(); + createTaskStmt.run(id, userId, question, depth, timestamp, timestamp); + appendEventStmt.run(id, 'queued', JSON.stringify({ depth }), timestamp); + return getTask(id); + }, + + updateTask(id, patch = {}) { + const normalized = { ...patch, updated_at: patch.updatedAt ?? now() }; + if (Array.isArray(normalized.plan)) { + normalized.plan_json = JSON.stringify(normalized.plan); + delete normalized.plan; + } + if (Object.hasOwn(normalized, 'completedAt')) { + normalized.completed_at = normalized.completedAt; + delete normalized.completedAt; + } + if (Object.hasOwn(normalized, 'updatedAt')) delete normalized.updatedAt; + const entries = Object.entries(normalized).filter(([key]) => TASK_COLUMNS.has(key)); + if (entries.length) { + const assignments = entries.map(([key]) => `${key} = ?`).join(', '); + db.prepare(`UPDATE research_tasks SET ${assignments} WHERE id = ?`) + .run(...entries.map(([, value]) => value), String(id)); + } + return getTask(id); + }, + + addSource(taskId, source = {}) { + addSourceStmt.run( + String(taskId), + String(source.url ?? ''), + String(source.title ?? '').slice(0, 1000), + String(source.snippet ?? '').slice(0, 4000), + String(source.content ?? '').slice(0, 100_000), + String(source.provider ?? 'unknown').slice(0, 100), + Number(source.score ?? 0) || 0, + String(source.query ?? '').slice(0, 1000), + now(), + ); + }, + + appendEvent(taskId, type, payload = {}) { + appendEventStmt.run(String(taskId), String(type), JSON.stringify(payload), now()); + }, + + saveMemory({ id, userId = null, topic, summary, sourceTaskId }) { + saveMemoryStmt.run(id, userId, topic, summary, sourceTaskId, now()); + }, + + getTask, + + listTasks(limit = 50) { + const bounded = Math.max(1, Math.min(200, Number(limit) || 50)); + return listTasksStmt.all(bounded).map(mapTask); + }, + + listMemories({ userId = null, limit = 50 } = {}) { + const bounded = Math.max(1, Math.min(200, Number(limit) || 50)); + const rows = userId + ? db.prepare('SELECT * FROM research_memories WHERE user_id = ? ORDER BY created_at DESC LIMIT ?') + .all(String(userId), bounded) + : db.prepare('SELECT * FROM research_memories ORDER BY created_at DESC LIMIT ?').all(bounded); + return rows.map((row) => ({ + id: row.id, + userId: row.user_id || null, + topic: row.topic, + summary: row.summary, + sourceTaskId: row.source_task_id, + createdAt: row.created_at, + })); + }, + + getStats() { + const rows = db.prepare('SELECT status, COUNT(*) AS count FROM research_tasks GROUP BY status').all(); + return Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])); + }, + + close() { + db.close(); + }, + }; +} diff --git a/deep-search.test.mjs b/deep-search.test.mjs new file mode 100644 index 0000000..a24134c --- /dev/null +++ b/deep-search.test.mjs @@ -0,0 +1,257 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + assertSafePublicUrl, + buildFallbackResearchPlan, + canonicalizeUrl, + createDeepSearchEngine, + extractEvidence, + htmlToPlainText, + readResearchSource, +} from './deep-search-engine.mjs'; +import { createDeepSearchHttpServer } from './deep-search-server.mjs'; +import { createDeepSearchStore } from './deep-search-store.mjs'; + +async function waitFor(check, { timeoutMs = 3000 } = {}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < timeoutMs) { + const value = await check(); + if (value) return value; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('Timed out waiting for condition'); +} + +function createMockEngine({ id = 'task-complete' } = {}) { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + const searches = []; + const engine = createDeepSearchEngine({ + store, + idFactory: () => id, + searchProvider: { + name: 'mock-search', + async search(query, { limit }) { + searches.push(query); + const slug = encodeURIComponent(query).slice(0, 60); + return Array.from({ length: Math.min(2, limit) }, (_, index) => ({ + title: `${query} source ${index + 1}`, + url: `https://example.com/${slug}/${index + 1}?utm_source=test`, + snippet: `Evidence about ${query}, with verifiable details and an independent source.`, + source: 'mock', + rank: index + 1, + })); + }, + }, + reader: async (url) => ({ + url, + title: `Document ${url}`, + content: 'This document provides verifiable evidence about the research question and explains the relevant facts in sufficient detail. It also identifies limitations and cites a primary source.', + contentType: 'text/html', + }), + }); + return { engine, store, searches }; +} + +test('Deep Search planner scales goals by depth and canonicalizes evidence URLs', () => { + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'quick').length, 3); + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'standard').length, 4); + assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'deep').length, 6); + assert.equal( + canonicalizeUrl('https://Example.com/report?utm_source=x&id=3#details'), + 'https://example.com/report?id=3', + ); + assert.match(htmlToPlainText('A

Hello & world

'), /Hello & world/); +}); + +test('Deep Search reader blocks private destinations and extracts public HTML', async () => { + await assert.rejects( + () => assertSafePublicUrl('http://127.0.0.1/private'), + /Unsafe source URL/, + ); + const document = await readResearchSource('https://example.com/article', { + lookup: async () => [{ address: '93.184.216.34', family: 4 }], + fetchImpl: async () => new Response( + 'Research
Useful evidence for the report.
', + { status: 200, headers: { 'content-type': 'text/html' } }, + ), + }); + assert.equal(document.title, 'Research'); + assert.match(document.content, /Useful evidence/); + assert.doesNotMatch(document.content, /ignore/); +}); + +test('Deep Search evidence extraction drops page chrome and HTML attribute noise', () => { + const evidence = extractEvidence({ + title: 'SearXNG privacy', + url: 'https://example.com/privacy', + content: [ + 'data-hydro-click="{\\"event_type\\":\\"authentication.click\\"}" class="HeaderMenu-link" this is navigation noise that must not become evidence.', + 'SearXNG protects search privacy by avoiding user profiling and by proxying requests to multiple upstream search engines without forwarding browser cookies.', + ].join('\n'), + }, { + question: 'How does SearXNG protect privacy?', + goal: 'Privacy design', + }); + assert.ok(evidence.length > 0); + assert.doesNotMatch(evidence.map((item) => item.claim).join('\n'), /data-hydro|HeaderMenu/); +}); + +test('Deep Search executes multi-round research and creates a cited report', async () => { + const { engine, store, searches } = createMockEngine(); + const started = engine.start({ question: '分析 AI Agent 市场', depth: 'standard', userId: 'user-1' }); + assert.deepEqual(started, { + taskId: 'task-complete', + status: 'queued', + service: 'tkmind-deep-search', + }); + const task = await waitFor(() => { + const current = engine.getTask(started.taskId); + return current.status === 'completed' ? current : null; + }); + assert.equal(task.progress, 100); + assert.equal(task.plan.length, 4); + assert.ok(searches.length >= 8); + assert.ok(task.sources.length > 0); + assert.match(task.report, /^# 分析 AI Agent 市场/m); + assert.match(task.report, /\[1\]/); + assert.match(task.report, /## 来源/); + assert.ok(task.events.some((event) => event.type === 'completed')); + assert.equal(store.listMemories({ userId: 'user-1' }).length, 1); + store.close(); +}); + +test('Deep Search uses a configured planner and citation-safe LLM report', async () => { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + let memoryPayload = null; + const llmReport = `# Model report + +This model-generated analysis is intentionally long enough to pass report validation and is grounded in the supplied evidence. It explains the finding, distinguishes evidence from inference, and preserves the required source citation. [1] + +## Sources + +1. Primary evidence [1]`; + const engine = createDeepSearchEngine({ + store, + idFactory: () => 'task-llm', + llm: { + async plan() { + return { + research_plan: [{ + goal: 'Primary evidence', + queries: ['official evidence', 'independent evidence'], + sources: ['web'], + }], + }; + }, + async synthesize() { + return llmReport; + }, + }, + memorySink: async (payload) => { + memoryPayload = payload; + }, + searchProvider: { + name: 'mock', + async search() { + return [{ + title: 'Primary source', + url: 'https://example.com/primary', + snippet: 'Primary evidence supports the tested claim.', + source: 'mock', + rank: 1, + }]; + }, + }, + reader: async (url) => ({ + url, + title: 'Primary source', + content: 'Primary evidence supports the tested claim with enough detail for a grounded research report and a verifiable citation.', + }), + }); + engine.start({ + question: 'Evaluate the tested claim', + depth: 'quick', + userId: 'memory-user', + }); + const task = await waitFor(() => { + const current = engine.getTask('task-llm'); + return current.status === 'completed' ? current : null; + }); + assert.equal(task.report, llmReport); + assert.equal(memoryPayload.userId, 'memory-user'); + assert.equal(memoryPayload.taskId, 'task-llm'); + assert.ok(task.events.some((event) => event.type === 'memory_saved')); + store.close(); +}); + +test('Deep Search supports cancellation while a provider is running', async () => { + const store = createDeepSearchStore({ databasePath: ':memory:' }); + const engine = createDeepSearchEngine({ + store, + idFactory: () => 'task-cancel', + searchProvider: { + name: 'slow', + search(_query, { signal }) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve([]), 2000); + signal.addEventListener('abort', () => { + clearTimeout(timeout); + const error = new Error('cancelled'); + error.name = 'AbortError'; + reject(error); + }, { once: true }); + }); + }, + }, + }); + engine.start({ question: 'A long-running investigation', depth: 'deep' }); + await waitFor(() => engine.getTask('task-cancel').status === 'researching'); + const cancelling = engine.cancel('task-cancel'); + assert.equal(cancelling.status, 'cancelling'); + const cancelled = await waitFor(() => { + const current = engine.getTask('task-cancel'); + return current.status === 'cancelled' ? current : null; + }); + assert.equal(cancelled.phase, 'cancelled'); + store.close(); +}); + +test('Deep Search HTTP service exposes health, async start, status, search, and auth', async (t) => { + const { engine, store } = createMockEngine({ id: 'task-http' }); + const server = createDeepSearchHttpServer({ engine, secret: 'local-secret', logger: { warn() {} } }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { + await new Promise((resolve) => server.close(resolve)); + store.close(); + }); + const address = server.address(); + const base = `http://127.0.0.1:${address.port}`; + const health = await fetch(`${base}/health`).then((response) => response.json()); + assert.equal(health.ok, true); + assert.equal((await fetch(`${base}/v1/research`)).status, 401); + const headers = { 'content-type': 'application/json', 'x-secret-key': 'local-secret' }; + const search = await fetch(`${base}/v1/search`, { + method: 'POST', + headers, + body: JSON.stringify({ query: 'Deep Search', limit: 2 }), + }).then((response) => response.json()); + assert.equal(search.results.length, 2); + const started = await fetch(`${base}/v1/research`, { + method: 'POST', + headers: { ...headers, 'x-user-id': 'user-a' }, + body: JSON.stringify({ question: 'Evaluate Deep Search', depth: 'quick' }), + }).then((response) => response.json()); + assert.equal(started.task_id, 'task-http'); + assert.equal((await fetch(`${base}/v1/research/${started.task_id}`, { + headers: { ...headers, 'x-user-id': 'user-b' }, + })).status, 403); + const completed = await waitFor(async () => { + const response = await fetch(`${base}/v1/research/${started.task_id}`, { + headers: { ...headers, 'x-user-id': 'user-a' }, + }); + const task = await response.json(); + return task.status === 'completed' ? task : null; + }); + assert.equal(completed.progress, 100); + assert.ok(completed.sources.every((source) => !Object.hasOwn(source, 'content'))); +}); diff --git a/docs/deep-search-service.md b/docs/deep-search-service.md new file mode 100644 index 0000000..376bdaa --- /dev/null +++ b/docs/deep-search-service.md @@ -0,0 +1,107 @@ +# TKMind Deep Search Service + +TKMind Deep Search is a local-first, independently deployable research orchestrator. MindSearch treats it as a `research-http` service, in the same way that SearXNG is registered as a search provider. + +The standalone runtime uses the built-in `node:sqlite` module and therefore requires a Node.js release that provides `DatabaseSync` (Node 22.5 or newer; the local verification used Node 26). + +## Runtime flow + +1. Validate and persist the task. +2. Build a depth-aware research plan. +3. Run bounded, multi-round searches through SearXNG. +4. Canonicalize, deduplicate, and rerank source URLs. +5. Resolve DNS and block private destinations before reading public pages. +6. Extract evidence and remove duplicate claims. +7. Generate a Markdown report with numbered citations. +8. Persist task state, sources, events, report, and research memory in SQLite. + +The planner and report writer use an OpenAI-compatible model when configured. If the model is unavailable, deterministic planning and citation-safe extractive reporting keep the service operational. + +`createDeepSearchEngine` also accepts an explicit `memorySink` callback. It is disabled by default; an in-process TKMind integration can inject `memoryV2.write` without giving the standalone service an unrestricted outbound memory endpoint. Memory sink failures are fail-open and recorded as task events. + +## Local run + +```bash +TKMIND_DEEP_SEARCH_SEARXNG_URL=http://127.0.0.1:8080/search \ +TKMIND_DEEP_SEARCH_DB=.deep-search/research.sqlite \ +npm run dev:deep-search +``` + +The server listens on `127.0.0.1:20100` by default. + +## API + +| Method | Path | Purpose | +| --- | --- | --- | +| `GET` | `/health` | Runtime health and task counts | +| `POST` | `/v1/search` | Synchronous normalized search | +| `POST` | `/v1/research` | Start an asynchronous research task | +| `GET` | `/v1/research` | List recent tasks | +| `GET` | `/v1/research/:id` | Get progress, evidence metadata, and report | +| `GET` | `/v1/research/:id/events` | Stream progress as server-sent events | +| `DELETE` | `/v1/research/:id` | Cancel a queued or running task | + +Start request: + +```json +{ + "question": "分析 AI Agent 市场", + "depth": "standard", + "userId": "optional-user-id" +} +``` + +Start response: + +```json +{ + "task_id": "uuid", + "status": "queued", + "service": "tkmind-deep-search" +} +``` + +## Configuration + +| Variable | Default | Meaning | +| --- | --- | --- | +| `TKMIND_DEEP_SEARCH_HOST` | `127.0.0.1` | Bind host | +| `TKMIND_DEEP_SEARCH_PORT` | `20100` | HTTP port | +| `TKMIND_DEEP_SEARCH_DB` | `.deep-search/research.sqlite` | SQLite database | +| `TKMIND_DEEP_SEARCH_SEARXNG_URL` | `http://127.0.0.1:8080/search` | Upstream search endpoint | +| `TKMIND_DEEP_SEARCH_SECRET` | empty | Optional `X-Secret-Key` required by non-health routes | +| `TKMIND_DEEP_SEARCH_LLM_URL` | empty | OpenAI-compatible chat-completions endpoint | +| `TKMIND_DEEP_SEARCH_LLM_API_KEY` | empty | Optional model API key | +| `TKMIND_DEEP_SEARCH_LLM_MODEL` | empty | Planner and report model | + +## Isolation and safety + +- The default bind address is loopback only. +- An optional service secret protects all non-health endpoints. +- MCP forwards the current TKMind user ID as `X-User-Id`. +- Task status, event, and cancellation routes enforce user ownership when a user ID is present. +- Source reading rejects credentials, loopback, link-local, private, multicast, and private-DNS destinations. +- Redirect targets are resolved and validated again. +- Request bodies, result counts, source content, timeouts, and task depths are bounded. + +## MindSearch registration + +The built-in disabled service is: + +```json +{ + "id": "deep-search", + "adapter": "research-http", + "endpoint": "http://127.0.0.1:20100", + "healthPath": "/health", + "enabled": false +} +``` + +After local verification, enable the service and MindSearch in `memindadm`, then select `deep-search` for the `research` route. New Goose sessions receive: + +- `tkmind_research` +- `tkmind_research_status` +- `tkmind_research_cancel` + +Enabling or publishing this service to production is a separate release action and is not performed by local development or tests. diff --git a/mindsearch-config.mjs b/mindsearch-config.mjs index 841c1d2..ba598b9 100644 --- a/mindsearch-config.mjs +++ b/mindsearch-config.mjs @@ -42,6 +42,18 @@ const BUILTIN_SERVICES = Object.freeze([ timeoutMs: 8000, priority: 80, }, + { + id: 'deep-search', + name: 'TKMind Deep Search', + kind: 'orchestrator', + adapter: 'research-http', + capabilities: ['search.web', 'research.plan', 'research.execute'], + endpoint: 'http://127.0.0.1:20100', + healthPath: '/health', + enabled: false, + timeoutMs: 120000, + priority: 100, + }, ]); export const MINDSEARCH_DEFAULT_CONFIG = Object.freeze({ @@ -50,7 +62,7 @@ export const MINDSEARCH_DEFAULT_CONFIG = Object.freeze({ providers: { searxng: false, github: false, reader: false }, settings: { searxngEndpoint: '', maxResults: 10, timeoutMs: 8000, readerMaxChars: 12000 }, services: BUILTIN_SERVICES, - routes: { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: '' }, + routes: { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: 'deep-search' }, }); const clone = (value) => JSON.parse(JSON.stringify(value)); @@ -115,7 +127,9 @@ function builtinsFromLegacy(input = {}) { endpoint: service.adapter === 'searxng' ? String(settings.searxngEndpoint ?? service.endpoint) : service.endpoint, - timeoutMs: settings.timeoutMs ?? service.timeoutMs, + timeoutMs: service.adapter === 'research-http' + ? service.timeoutMs + : (settings.timeoutMs ?? service.timeoutMs), }, service)); } @@ -185,7 +199,12 @@ export async function probeMindSearchService(service, { fetchImpl = fetch } = {} try { const response = await fetchImpl(url, { signal: AbortSignal.timeout(normalized.timeoutMs), - headers: { accept: 'application/json,text/plain' }, + headers: { + accept: 'application/json,text/plain', + ...(normalized.adapter === 'research-http' && process.env.TKMIND_DEEP_SEARCH_SECRET + ? { 'x-secret-key': process.env.TKMIND_DEEP_SEARCH_SECRET } + : {}), + }, }); let resultCount = null; if (normalized.adapter === 'searxng' && response.ok) { diff --git a/mindsearch-providers.mjs b/mindsearch-providers.mjs index dfa53ef..e27bc6c 100644 --- a/mindsearch-providers.mjs +++ b/mindsearch-providers.mjs @@ -1,11 +1,21 @@ import { isSafeHttpUrl, normalizeSearchResult } from './search-capability.mjs'; import { isValidSearchServiceEndpoint } from './mindsearch-config.mjs'; -export async function searchSearxng(query, { limit = 10, endpoint = process.env.TKMIND_SEARCH_SEARXNG_URL, fetchImpl = fetch } = {}) { +export async function searchSearxng(query, { + limit = 10, + endpoint = process.env.TKMIND_SEARCH_SEARXNG_URL, + fetchImpl = fetch, + timeoutMs = Number(process.env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000), + signal, +} = {}) { if (!endpoint) throw new Error('SearXNG endpoint is not configured'); try { if (!['http:', 'https:'].includes(new URL(endpoint).protocol)) throw new Error('unsupported protocol'); } catch { throw new Error('SearXNG endpoint is not configured safely'); } const url = new URL(endpoint); url.searchParams.set('q', query); url.searchParams.set('format', 'json'); - const response = await fetchImpl(url, { signal: AbortSignal.timeout(Number(process.env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000)), headers: { accept: 'application/json' } }); + const timeout = AbortSignal.timeout(timeoutMs); + const response = await fetchImpl(url, { + signal: signal ? AbortSignal.any([signal, timeout]) : timeout, + headers: { accept: 'application/json' }, + }); if (!response.ok) throw new Error(`SearXNG returned ${response.status}`); const body = await response.json(); return (Array.isArray(body.results) ? body.results : []).slice(0, limit).map((item, index) => normalizeSearchResult({ ...item, source: 'searxng' }, index)); @@ -37,11 +47,21 @@ function serviceUrl(endpoint, path) { return new URL(path.replace(/^\//, ''), base); } -async function postServiceJson(endpoint, path, payload, { timeoutMs = 30000, fetchImpl = fetch } = {}) { +async function postServiceJson(endpoint, path, payload, { + timeoutMs = 30000, + fetchImpl = fetch, + userId = null, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { const response = await fetchImpl(serviceUrl(endpoint, path), { method: 'POST', signal: AbortSignal.timeout(timeoutMs), - headers: { accept: 'application/json', 'content-type': 'application/json' }, + headers: { + accept: 'application/json', + 'content-type': 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, body: JSON.stringify(payload), }); if (!response.ok) throw new Error(`Search service returned ${response.status}`); @@ -54,8 +74,13 @@ export async function searchResearchService(query, { limit = 10, timeoutMs = 30000, fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, } = {}) { - const body = await postServiceJson(endpoint, '/v1/search', { query, type, limit }, { timeoutMs, fetchImpl }); + const body = await postServiceJson(endpoint, '/v1/search', { query, type, limit }, { + timeoutMs, + fetchImpl, + secret, + }); return (Array.isArray(body.results) ? body.results : []) .slice(0, limit) .map((item, index) => normalizeSearchResult({ ...item, source: item.source || 'research-service' }, index)); @@ -64,10 +89,16 @@ export async function searchResearchService(query, { export async function startResearchTask(question, { endpoint, depth = 'standard', + userId = null, timeoutMs = 30000, fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, } = {}) { - const body = await postServiceJson(endpoint, '/v1/research', { question, depth }, { timeoutMs, fetchImpl }); + const body = await postServiceJson(endpoint, '/v1/research', { + question, + depth, + ...(userId ? { userId } : {}), + }, { timeoutMs, fetchImpl, userId, secret }); if (!body || typeof body !== 'object' || !body.task_id) throw new Error('Research service returned an invalid task'); return { taskId: String(body.task_id), @@ -75,3 +106,44 @@ export async function startResearchTask(question, { service: String(body.service ?? 'research-service'), }; } + +export async function getResearchTask(taskId, { + endpoint, + userId = null, + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), { + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, + }); + if (response.status === 404) throw new Error('Research task was not found'); + if (!response.ok) throw new Error(`Search service returned ${response.status}`); + return response.json(); +} + +export async function cancelResearchTask(taskId, { + endpoint, + userId = null, + timeoutMs = 30000, + fetchImpl = fetch, + secret = process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), { + method: 'DELETE', + signal: AbortSignal.timeout(timeoutMs), + headers: { + accept: 'application/json', + ...(userId ? { 'x-user-id': String(userId) } : {}), + ...(secret ? { 'x-secret-key': String(secret) } : {}), + }, + }); + if (response.status === 404) throw new Error('Research task was not found'); + if (!response.ok) throw new Error(`Search service returned ${response.status}`); + return response.json(); +} diff --git a/mindsearch.test.mjs b/mindsearch.test.mjs index 41301c3..7980bc8 100644 --- a/mindsearch.test.mjs +++ b/mindsearch.test.mjs @@ -9,6 +9,8 @@ import { } from './mindsearch-config.mjs'; import { buildAgentExtensionPolicy, DEFAULT_USER_CAPABILITIES } from './capabilities.mjs'; import { + cancelResearchTask, + getResearchTask, searchGithubCode, searchResearchService, searchSearxng, @@ -22,7 +24,8 @@ test('MindSearch defaults to disabled and normalizes unsafe modes', () => { assert.equal(config.mode, 'off'); assert.deepEqual(config.providers, { searxng: false, github: true, reader: false }); assert.equal(config.services.find((service) => service.id === 'github').enabled, true); - assert.deepEqual(config.routes, { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: '' }); + assert.deepEqual(config.routes, { web: 'searxng', news: 'searxng', code: 'github', read: 'reader', research: 'deep-search' }); + assert.equal(config.services.find((service) => service.id === 'deep-search').enabled, false); }); test('MindSearch service registry accepts research orchestrators and rejects unsafe endpoints', () => { @@ -89,6 +92,20 @@ test('MindSearch never changes legacy web extension and is gated by capability/c assert.equal(policy.extensionOverrides.some((ext) => ext.name === 'tkmind-search'), false); policy = buildAgentExtensionPolicy({ ...base, search_external: true }, { mindSearchConfig: { enabled: true, mode: 'assist', providers: {} } }); assert.ok(policy.extensionOverrides.some((ext) => ext.name === 'tkmind-search')); + policy = buildAgentExtensionPolicy({ ...base, search_external: true }, { + userId: 'user-123', + mindSearchConfig: { + enabled: true, + mode: 'assist', + providers: {}, + services: [{ id: 'deep-search', enabled: true, adapter: 'research-http' }], + routes: { research: 'deep-search' }, + }, + }); + const extension = policy.extensionOverrides.find((ext) => ext.name === 'tkmind-search'); + assert.equal(extension.envs.TKMIND_SEARCH_USER_ID, 'user-123'); + assert.ok(extension.available_tools.includes('tkmind_research_status')); + assert.ok(extension.available_tools.includes('tkmind_research_cancel')); }); test('SearXNG adapter normalizes provider results without requiring a live network', async () => { @@ -111,12 +128,40 @@ test('Research service adapter uses the stable HTTP contract', async () => { if (String(url).endsWith('/v1/research')) { return { ok: true, json: async () => ({ task_id: 'task-1', status: 'running' }) }; } + if (String(url).endsWith('/v1/research/task-1')) { + return { + ok: true, + status: 200, + json: async () => ({ + id: 'task-1', + status: options?.method === 'DELETE' ? 'cancelling' : 'researching', + progress: 55, + }), + }; + } return { ok: true, json: async () => ({ results: [{ title: 'Report', url: 'https://example.com/report', snippet: 'evidence' }] }) }; }; const results = await searchResearchService('AI Agent', { endpoint: 'http://research.local', limit: 5, fetchImpl }); - const task = await startResearchTask('Analyze AI Agent', { endpoint: 'http://research.local', depth: 'deep', fetchImpl }); + const task = await startResearchTask('Analyze AI Agent', { + endpoint: 'http://research.local', + depth: 'deep', + userId: 'user-1', + secret: 'secret-1', + fetchImpl, + }); + const status = await getResearchTask('task-1', { endpoint: 'http://research.local', fetchImpl }); + const cancelled = await cancelResearchTask('task-1', { endpoint: 'http://research.local', fetchImpl }); assert.equal(results[0].source, 'research-service'); assert.deepEqual(task, { taskId: 'task-1', status: 'running', service: 'research-service' }); + assert.equal(status.status, 'researching'); + assert.equal(cancelled.status, 'cancelling'); assert.equal(JSON.parse(requests[0].options.body).query, 'AI Agent'); - assert.deepEqual(JSON.parse(requests[1].options.body), { question: 'Analyze AI Agent', depth: 'deep' }); + assert.deepEqual(JSON.parse(requests[1].options.body), { + question: 'Analyze AI Agent', + depth: 'deep', + userId: 'user-1', + }); + assert.equal(requests[1].options.headers['x-user-id'], 'user-1'); + assert.equal(requests[1].options.headers['x-secret-key'], 'secret-1'); + assert.equal(requests[3].options.method, 'DELETE'); }); diff --git a/package.json b/package.json index 9733ec8..738eb46 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dev:vite": "vite", "dev:server": "node server.mjs", "dev:adm": "node admin-server.mjs", + "dev:deep-search": "node deep-search-server.mjs", "dev:plaza-express": "node plaza-server.mjs", "dev:ops": "cd ops && npm run dev", "launch:executor": "node scripts/launch-executor.mjs", @@ -61,6 +62,7 @@ "test:scenario:john4-diet": "node scripts/run-scenario-test.mjs --scenario john4-children-hobby-diet-update", "test": "node --test auth.test.mjs asr-proxy.test.mjs billing.test.mjs billing-token-state.test.mjs billing-recharge.test.mjs wechat-pay.test.mjs wechat-oauth.test.mjs wechat-mp.test.mjs wechat-media.test.mjs wechat/image-generation-policy.test.mjs wechat/verify/generated-thumbnail.test.mjs schedule-intent.test.mjs schedule-reminder-worker.test.mjs capabilities.test.mjs policies.test.mjs chat-skills.test.mjs chat-intent-router.test.mjs chat-finish-sync.test.mjs chat-agent-run-gate.test.mjs conversation-display.test.mjs user-publish.test.mjs user-memory-profile.test.mjs skills-registry.test.mjs skill-runtime-policy.test.mjs excel-analyst.test.mjs agent-run-gateway.test.mjs agent-run-routes.test.mjs session-broker.test.mjs sse-event-taxonomy.test.mjs goosed-proxy-boundary.test.mjs agent-run-stream.test.mjs mindspace-h5-html-finish-guard.test.mjs admin-routes.test.mjs image-make-admin-config.test.mjs asset-gateway.test.mjs image-make-client.test.mjs mindspace-image-generation.test.mjs mindspace-image-generation-routes.test.mjs mindspace-image-review.test.mjs mindspace-run-public-html-scope.test.mjs direct-chat-service.test.mjs tool-gateway.test.mjs mindspace.test.mjs mindspace-scan.test.mjs mindspace-assets.test.mjs mindspace-local-runtime-services.test.mjs mindspace-local-server-adapter.test.mjs mindspace-public-asset-token.test.mjs mindspace-remote-server-adapter.test.mjs mindspace-server-adapter.test.mjs mindspace-pages.test.mjs mindspace-page-sync-service.test.mjs public-site-bases.test.mjs mindspace-html-download-links.test.mjs mindspace-page-purge.test.mjs mindspace-public-delivery.test.mjs mindspace-public-page-context.test.mjs mindspace-published-page-csp.test.mjs mindspace-published-script-localize.test.mjs agent-run-deliverable-check.test.mjs mindspace-public-route.test.mjs mindspace-publications.test.mjs mindspace-public-links.test.mjs mindspace-chat-save.test.mjs mindspace-chat-docx-package.test.mjs mindspace-public-finish-sync.test.mjs mindspace-chat-context.test.mjs mindspace-canonical-url.test.mjs mindspace-conversation-package.test.mjs mindspace-conversation-package-backfill.test.mjs mindspace-conversation-package-public-html.test.mjs mindspace-conversation-package-verify.test.mjs mindspace-conversation-package-registry.test.mjs mindspace-conversation-package-routes.test.mjs mindspace-conversation-package-store.test.mjs mindspace-conversation-schema.test.mjs mindspace-runtime-config.test.mjs mindspace-config.test.mjs mindspace-analytics.test.mjs mindspace-service.test.mjs mindspace-storage-adapter.test.mjs mindspace-content-scan.test.mjs mindspace-html-localize.test.mjs mindspace-visual-editor.test.mjs mindspace-cleanup.test.mjs mindspace-thumbnails.test.mjs mindspace-workspace-thumbnails.test.mjs mindspace-workspace-sync.test.mjs mindspace-asset-preview.test.mjs mindspace-agent-jobs.test.mjs mindspace-agent-runner.test.mjs mindspace-sandbox-mcp.test.mjs mindspace-userdata-postgres.test.mjs postgres-user-data-space-service.test.mjs user-data-space-service.test.mjs page-data-routes.test.mjs page-access-policy.test.mjs page-access-visitor.test.mjs page-data-public-service.test.mjs page-data-integration.test.mjs page-data-log-store.test.mjs page-data-ops.test.mjs page-data-session-store.test.mjs page-data-browser-client.test.mjs page-data-policy-index.test.mjs message-stream.test.mjs mindspace-service/mindspace-rpc-server.test.mjs plaza-posts.test.mjs plaza-interactions.test.mjs plaza-algorithm.test.mjs plaza-seo.test.mjs plaza-ops.test.mjs user-auth.test.mjs llm-providers.test.mjs admin-guard.test.mjs user-feedback.test.mjs memory-v2.test.mjs memory-v2-admin-config.test.mjs memory-v2-lifecycle.test.mjs memory-v2-adapter-scaffold.test.mjs memory-v2-backend-contract.test.mjs memory-v2-health.test.mjs memory-v2-runtime.test.mjs memory-v2-plugin-backends.test.mjs memory-v2-pgvector.test.mjs memory-v2-pgvector-schema.test.mjs memory-v2-pgvector-backfill.test.mjs memory-v2-pgvector-smoke.test.mjs memory-v2-qdrant.test.mjs memory-v2-weaviate.test.mjs memory-v2-mem0.test.mjs memory-v2-letta.test.mjs memory-v2-external-adapters.test.mjs scripts/embed-memory-v2-local-hash.test.mjs scripts/check-memory-v2-app-canary.test.mjs scripts/check-memory-v2-config-gaps.test.mjs scripts/check-memory-v2-contracts.test.mjs scripts/check-memory-v2-health.test.mjs scripts/check-memory-v2-session-flow.test.mjs scripts/check-memory-v2-stack.test.mjs scripts/setup-memory-v2-pgvector-schema.test.mjs scripts/backfill-memory-v2-pgvector.test.mjs scripts/scaffold-memory-v2-backend.test.mjs scripts/smoke-memory-v2-pgvector.test.mjs scripts/smoke-memory-v2-qdrant.test.mjs scripts/smoke-memory-v2-external.test.mjs scripts/mock-memory-v2-services.test.mjs", "test:episodic-memory": "node --test episodic-memory.test.mjs direct-chat-service.test.mjs chat-intent-router.test.mjs", + "test:deep-search": "node --test deep-search.test.mjs mindsearch.test.mjs", "test:image-review": "node --test mindspace-image-review.test.mjs mindspace-image-generation.test.mjs", "test:mindspace-service": "node --test mindspace-service/mindspace-rpc-server.test.mjs", "verify:chat-finish-sync": "node scripts/verify-chat-finish-sync.mjs", diff --git a/tkmind-search-mcp.mjs b/tkmind-search-mcp.mjs index 553a4bd..20a6bfb 100644 --- a/tkmind-search-mcp.mjs +++ b/tkmind-search-mcp.mjs @@ -2,6 +2,8 @@ import readline from 'node:readline'; import { normalizeMindSearchConfig, resolveMindSearchService } from './mindsearch-config.mjs'; import { SEARCH_ERROR_CODES, validateSearchRequest } from './search-capability.mjs'; import { + cancelResearchTask, + getResearchTask, readSafeUrl, searchGithubCode, searchResearchService, @@ -46,6 +48,26 @@ if (resolveMindSearchService(config, 'research')) { required: ['question'], }, }); + tools.push( + { + name: 'tkmind_research_status', + description: 'Get Deep Search progress, evidence sources, and the completed report.', + inputSchema: { + type: 'object', + properties: { task_id: { type: 'string' } }, + required: ['task_id'], + }, + }, + { + name: 'tkmind_research_cancel', + description: 'Cancel a queued or running Deep Search task.', + inputSchema: { + type: 'object', + properties: { task_id: { type: 'string' } }, + required: ['task_id'], + }, + }, + ); } function response(id, result) { process.stdout.write(`${JSON.stringify({ jsonrpc: '2.0', id, result })}\n`); } @@ -81,10 +103,32 @@ function handle(message) { const depth = ['quick', 'standard', 'deep'].includes(params.arguments?.depth) ? params.arguments.depth : 'standard'; const service = resolveMindSearchService(config, 'research'); if (service?.adapter !== 'research-http') return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No research service is configured.'); - return startResearchTask(question, { endpoint: service.endpoint, depth, timeoutMs: service.timeoutMs }) + return startResearchTask(question, { + endpoint: service.endpoint, + depth, + userId: process.env.TKMIND_SEARCH_USER_ID || null, + timeoutMs: service.timeoutMs, + }) .then((result) => response(id, { content: [{ type: 'text', text: JSON.stringify(result) }], structuredContent: result })) .catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); } + if (name === 'tkmind_research_status' || name === 'tkmind_research_cancel') { + const taskId = String(params.arguments?.task_id ?? '').trim(); + if (!taskId || taskId.length > 200) return error(id, SEARCH_ERROR_CODES.INVALID_REQUEST, 'task_id must be 1-200 characters'); + const service = resolveMindSearchService(config, 'research'); + if (service?.adapter !== 'research-http') return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No research service is configured.'); + const operation = name === 'tkmind_research_cancel' ? cancelResearchTask : getResearchTask; + return operation(taskId, { + endpoint: service.endpoint, + userId: process.env.TKMIND_SEARCH_USER_ID || null, + timeoutMs: service.timeoutMs, + }) + .then((result) => response(id, { + content: [{ type: 'text', text: result.report || JSON.stringify(result) }], + structuredContent: result, + })) + .catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + } return error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, 'No reader provider is configured.'); } diff --git a/user-auth.mjs b/user-auth.mjs index a241815..e5022d2 100644 --- a/user-auth.mjs +++ b/user-auth.mjs @@ -1868,7 +1868,12 @@ export function createUserAuth(pool, options = {}) { await syncUserSkillsForUser(user); if (capabilityState.unrestricted) { return { - ...buildAgentExtensionPolicy(capabilityState.capabilities, { unrestricted: true, toolMode, mindSearchConfig }), + ...buildAgentExtensionPolicy(capabilityState.capabilities, { + unrestricted: true, + toolMode, + mindSearchConfig, + userId: user.id, + }), policies: {}, unrestricted: true, toolMode, @@ -1933,6 +1938,7 @@ export function createUserAuth(pool, options = {}) { sandboxMcp, toolMode, mindSearchConfig, + userId: user.id, }), capabilities: effectiveCapabilities, policies: policyState.policies,