import assert from 'node:assert/strict';
import test from 'node:test';
import {
assertSafePublicUrl,
buildFallbackResearchPlan,
canonicalizeUrl,
createDeepSearchEngine,
createPortalGatewayResearchLlmResolver,
extractEvidence,
htmlToPlainText,
readResearchSource,
} from './deep-search-engine.mjs';
import { executeDeepSearchLlmGateway } from './deep-search-llm-gateway.mjs';
import { createDeepSearchHttpServer } from './deep-search-server.mjs';
import { createDeepSearchStore } from './deep-search-store.mjs';
async function waitFor(check, { timeoutMs = 3000 } = {}) {
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
const value = await check();
if (value) return value;
await new Promise((resolve) => setTimeout(resolve, 10));
}
throw new Error('Timed out waiting for condition');
}
function createMockEngine({ id = 'task-complete' } = {}) {
const store = createDeepSearchStore({ databasePath: ':memory:' });
const searches = [];
const engine = createDeepSearchEngine({
store,
idFactory: () => id,
searchProvider: {
name: 'mock-search',
async search(query, { limit }) {
searches.push(query);
const slug = encodeURIComponent(query).slice(0, 60);
return Array.from({ length: Math.min(2, limit) }, (_, index) => ({
title: `${query} source ${index + 1}`,
url: `https://example.com/${slug}/${index + 1}?utm_source=test`,
snippet: `Evidence about ${query}, with verifiable details and an independent source.`,
source: 'mock',
rank: index + 1,
}));
},
},
reader: async (url) => ({
url,
title: `Document ${url}`,
content: 'This document provides verifiable evidence about the research question and explains the relevant facts in sufficient detail. It also identifies limitations and cites a primary source.',
contentType: 'text/html',
}),
});
return { engine, store, searches };
}
test('Deep Search planner scales goals by depth and canonicalizes evidence URLs', () => {
assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'quick').length, 3);
assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'standard').length, 4);
assert.equal(buildFallbackResearchPlan('AI Agent 市场', 'deep').length, 6);
assert.equal(
canonicalizeUrl('https://Example.com/report?utm_source=x&id=3#details'),
'https://example.com/report?id=3',
);
assert.match(htmlToPlainText('AHello & world
'), /Hello & world/);
});
test('Deep Search reader blocks private destinations and extracts public HTML', async () => {
await assert.rejects(
() => assertSafePublicUrl('http://127.0.0.1/private'),
/Unsafe source URL/,
);
const document = await readResearchSource('https://example.com/article', {
lookup: async () => [{ address: '93.184.216.34', family: 4 }],
fetchImpl: async () => new Response(
'ResearchUseful evidence for the report.',
{ status: 200, headers: { 'content-type': 'text/html' } },
),
});
assert.equal(document.title, 'Research');
assert.match(document.content, /Useful evidence/);
assert.doesNotMatch(document.content, /ignore/);
});
test('Deep Search evidence extraction drops page chrome and HTML attribute noise', () => {
const evidence = extractEvidence({
title: 'SearXNG privacy',
url: 'https://example.com/privacy',
content: [
'data-hydro-click="{\\"event_type\\":\\"authentication.click\\"}" class="HeaderMenu-link" this is navigation noise that must not become evidence.',
'SearXNG protects search privacy by avoiding user profiling and by proxying requests to multiple upstream search engines without forwarding browser cookies.',
].join('\n'),
}, {
question: 'How does SearXNG protect privacy?',
goal: 'Privacy design',
});
assert.ok(evidence.length > 0);
assert.doesNotMatch(evidence.map((item) => item.claim).join('\n'), /data-hydro|HeaderMenu/);
});
test('Deep Search executes multi-round research and creates a cited report', async () => {
const { engine, store, searches } = createMockEngine();
const started = engine.start({ question: '分析 AI Agent 市场', depth: 'standard', userId: 'user-1' });
assert.deepEqual(started, {
taskId: 'task-complete',
status: 'queued',
service: 'tkmind-deep-search',
});
const task = await waitFor(() => {
const current = engine.getTask(started.taskId);
return current.status === 'completed' ? current : null;
});
assert.equal(task.progress, 100);
assert.equal(task.plan.length, 4);
assert.ok(searches.length >= 8);
assert.ok(task.sources.length > 0);
assert.match(task.report, /^# 分析 AI Agent 市场/m);
assert.match(task.report, /\[1\]/);
assert.match(task.report, /## 来源/);
assert.ok(task.events.some((event) => event.type === 'completed'));
assert.equal(store.listMemories({ userId: 'user-1' }).length, 1);
store.close();
});
test('Deep Search uses a configured planner and citation-safe LLM report', async () => {
const store = createDeepSearchStore({ databasePath: ':memory:' });
let memoryPayload = null;
const llmReport = `# Model report
This model-generated analysis is intentionally long enough to pass report validation and is grounded in the supplied evidence. It explains the finding, distinguishes evidence from inference, and preserves the required source citation. [1]
## Sources
1. Primary evidence [1]`;
const engine = createDeepSearchEngine({
store,
idFactory: () => 'task-llm',
llmResolver: async ({ providerKeyId, model }) => {
assert.equal(providerKeyId, 'provider-key-1');
assert.equal(model, 'research-model');
return {
async plan() {
return {
research_plan: [{
goal: 'Primary evidence',
queries: ['official evidence', 'independent evidence'],
sources: ['web'],
}],
};
},
async synthesize() {
return llmReport;
},
};
},
memorySink: async (payload) => {
memoryPayload = payload;
},
searchProvider: {
name: 'mock',
async search() {
return [{
title: 'Primary source',
url: 'https://example.com/primary',
snippet: 'Primary evidence supports the tested claim.',
source: 'mock',
rank: 1,
}];
},
},
reader: async (url) => ({
url,
title: 'Primary source',
content: 'Primary evidence supports the tested claim with enough detail for a grounded research report and a verifiable citation.',
}),
});
engine.start({
question: 'Evaluate the tested claim',
depth: 'quick',
userId: 'memory-user',
llmProviderKeyId: 'provider-key-1',
llmModel: 'research-model',
});
const task = await waitFor(() => {
const current = engine.getTask('task-llm');
return current.status === 'completed' ? current : null;
});
assert.equal(task.report, llmReport);
assert.equal(task.llmProviderKeyId, 'provider-key-1');
assert.equal(task.llmModel, 'research-model');
assert.ok(task.events.some((event) => event.type === 'llm_selected'));
assert.equal(memoryPayload.userId, 'memory-user');
assert.equal(memoryPayload.taskId, 'task-llm');
assert.ok(task.events.some((event) => event.type === 'memory_saved'));
store.close();
});
test('Deep Search portal LLM gateway keeps provider secrets inside the portal', async () => {
let request = null;
const unauthorized = await executeDeepSearchLlmGateway({
llmProviderService: {},
expectedSecret: 'expected',
providedSecret: 'wrong',
input: {},
});
assert.equal(unauthorized.status, 401);
const authorized = await executeDeepSearchLlmGateway({
expectedSecret: 'expected',
providedSecret: 'expected',
input: {
providerKeyId: 'key-1',
model: 'model-1',
messages: [{ role: 'user', content: 'Plan research' }],
},
llmProviderService: {
async createChatCompletion(input) {
request = input;
return {
ok: true,
providerKeyId: 'key-1',
providerId: 'custom',
model: 'model-1',
reply: '{"research_plan":[]}',
};
},
},
});
assert.equal(authorized.status, 200);
assert.equal(request.providerKeyId, 'key-1');
assert.equal(request.model, 'model-1');
const resolver = createPortalGatewayResearchLlmResolver({
endpoint: 'http://portal.local/api/internal/deep-search/llm',
secret: 'expected',
fetchImpl: async (_url, options) => {
const body = JSON.parse(options.body);
assert.equal(options.headers.authorization, 'Bearer expected');
assert.equal(body.providerKeyId, 'key-1');
return new Response(JSON.stringify({ ok: true, reply: '{"research_plan":[]}' }), {
status: 200,
headers: { 'content-type': 'application/json' },
});
},
});
assert.equal(resolver.configured, true);
const resolved = await resolver({ providerKeyId: 'key-1', model: 'model-1' });
assert.deepEqual(await resolved.plan('Question', 'quick'), { research_plan: [] });
});
test('Deep Search supports cancellation while a provider is running', async () => {
const store = createDeepSearchStore({ databasePath: ':memory:' });
const engine = createDeepSearchEngine({
store,
idFactory: () => 'task-cancel',
searchProvider: {
name: 'slow',
search(_query, { signal }) {
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => resolve([]), 2000);
signal.addEventListener('abort', () => {
clearTimeout(timeout);
const error = new Error('cancelled');
error.name = 'AbortError';
reject(error);
}, { once: true });
});
},
},
});
engine.start({ question: 'A long-running investigation', depth: 'deep' });
await waitFor(() => engine.getTask('task-cancel').status === 'researching');
const cancelling = engine.cancel('task-cancel');
assert.equal(cancelling.status, 'cancelling');
const cancelled = await waitFor(() => {
const current = engine.getTask('task-cancel');
return current.status === 'cancelled' ? current : null;
});
assert.equal(cancelled.phase, 'cancelled');
store.close();
});
test('Deep Search HTTP service exposes health, async start, status, search, and auth', async (t) => {
const { engine, store } = createMockEngine({ id: 'task-http' });
let githubRequest;
const server = createDeepSearchHttpServer({
engine,
secret: 'local-secret',
githubToken: 'github-secret-token',
githubSearch: async (query, options) => {
githubRequest = { query, options };
return [{
title: 'tkmind/code:server.mjs',
url: 'https://github.com/tkmind/code/blob/main/server.mjs',
snippet: 'server',
source: 'github',
rank: 1,
}];
},
logger: { warn() {} },
});
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
t.after(async () => {
await new Promise((resolve) => server.close(resolve));
store.close();
});
const address = server.address();
const base = `http://127.0.0.1:${address.port}`;
const health = await fetch(`${base}/health`).then((response) => response.json());
assert.equal(health.ok, true);
assert.equal(health.providers.github.configured, true);
assert.equal((await fetch(`${base}/v1/research`)).status, 401);
const headers = { 'content-type': 'application/json', 'x-secret-key': 'local-secret' };
const github = await fetch(`${base}/v1/providers/github/search`, {
method: 'POST',
headers,
body: JSON.stringify({ query: 'repo:tkmind/code server', limit: 3 }),
}).then((response) => response.json());
assert.equal(github.results.length, 1);
assert.equal(githubRequest.options.token, 'github-secret-token');
assert.equal(githubRequest.options.limit, 3);
const search = await fetch(`${base}/v1/search`, {
method: 'POST',
headers,
body: JSON.stringify({ query: 'Deep Search', limit: 2 }),
}).then((response) => response.json());
assert.equal(search.results.length, 2);
const started = await fetch(`${base}/v1/research`, {
method: 'POST',
headers: { ...headers, 'x-user-id': 'user-a' },
body: JSON.stringify({
question: 'Evaluate Deep Search',
depth: 'quick',
llm_provider_key_id: 'key-http',
llm_model: 'model-http',
}),
}).then((response) => response.json());
assert.equal(started.task_id, 'task-http');
assert.equal((await fetch(`${base}/v1/research/${started.task_id}`, {
headers: { ...headers, 'x-user-id': 'user-b' },
})).status, 403);
const completed = await waitFor(async () => {
const response = await fetch(`${base}/v1/research/${started.task_id}`, {
headers: { ...headers, 'x-user-id': 'user-a' },
});
const task = await response.json();
return task.status === 'completed' ? task : null;
});
assert.equal(completed.progress, 100);
assert.equal(completed.llmProviderKeyId, 'key-http');
assert.equal(completed.llmModel, 'model-http');
assert.ok(completed.sources.every((source) => !Object.hasOwn(source, 'content')));
});