memind/mindsearch-providers.mjs

import { isSafeHttpUrl, normalizeSearchResult } from './search-capability.mjs';
import { isValidSearchServiceEndpoint } from './mindsearch-config.mjs';

export async function searchSearxng(query, {
  limit = 10,
  endpoint = process.env.TKMIND_SEARCH_SEARXNG_URL,
  fetchImpl = fetch,
  timeoutMs = Number(process.env.TKMIND_SEARCH_TIMEOUT_MS ?? 8000),
  signal,
  categories = '',
  language = '',
  engines = '',
  safesearch = null,
} = {}) {
  if (!endpoint) throw new Error('SearXNG endpoint is not configured');
  try { if (!['http:', 'https:'].includes(new URL(endpoint).protocol)) throw new Error('unsupported protocol'); } catch { throw new Error('SearXNG endpoint is not configured safely'); }
  const url = new URL(endpoint);
  url.searchParams.set('q', query);
  url.searchParams.set('format', 'json');
  if (categories) url.searchParams.set('categories', String(categories));
  if (language) url.searchParams.set('language', String(language));
  if (engines) url.searchParams.set('engines', String(engines));
  if (safesearch != null) url.searchParams.set('safesearch', String(safesearch));
  const timeout = AbortSignal.timeout(timeoutMs);
  const response = await fetchImpl(url, {
    signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
    headers: { accept: 'application/json' },
  });
  if (!response.ok) throw new Error(`SearXNG returned ${response.status}`);
  const body = await response.json();
  return (Array.isArray(body.results) ? body.results : []).slice(0, limit).map((item, index) => normalizeSearchResult({ ...item, source: 'searxng' }, index));
}

export async function readSafeUrl(target, { fetchImpl = fetch } = {}) {
  if (!isSafeHttpUrl(target)) throw new Error('unsafe URL');
  const response = await fetchImpl(target, { signal: AbortSignal.timeout(8000), headers: { accept: 'text/html,text/plain' } });
  if (!response.ok) throw new Error(`reader returned ${response.status}`);
  return { url: target, content: (await response.text()).slice(0, Number(process.env.TKMIND_SEARCH_READER_MAX_CHARS ?? 12000)) };
}

export async function searchGithubCode(query, { limit = 10, token = process.env.GITHUB_TOKEN, endpoint = process.env.TKMIND_SEARCH_GITHUB_URL || 'https://api.github.com/search/code', fetchImpl = fetch } = {}) {
  const url = new URL(endpoint);
  if (url.protocol !== 'https:' || url.hostname !== 'api.github.com') throw new Error('GitHub endpoint is not configured safely');
  url.searchParams.set('q', query);
  url.searchParams.set('per_page', String(Math.min(20, limit)));
  const headers = { accept: 'application/vnd.github+json', 'user-agent': 'tkmind-search' };
  if (token) headers.authorization = `Bearer ${token}`;
  const response = await fetchImpl(url, { headers, signal: AbortSignal.timeout(8000) });
  if (!response.ok) throw new Error(`GitHub returned ${response.status}`);
  const body = await response.json();
  return (Array.isArray(body.items) ? body.items : []).slice(0, limit).map((item, index) => normalizeSearchResult({ title: `${item.repository?.full_name ?? ''}:${item.path ?? item.name ?? ''}`, url: item.html_url, snippet: item.repository?.description ?? '', source: 'github' }, index));
}

function serviceUrl(endpoint, path) {
  if (!isValidSearchServiceEndpoint(endpoint, { allowEmpty: false })) throw new Error('Search service endpoint is not configured safely');
  const base = endpoint.endsWith('/') ? endpoint : `${endpoint}/`;
  return new URL(path.replace(/^\//, ''), base);
}

async function postServiceJson(endpoint, path, payload, {
  timeoutMs = 30000,
  fetchImpl = fetch,
  userId = null,
  secret = process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const response = await fetchImpl(serviceUrl(endpoint, path), {
    method: 'POST',
    signal: AbortSignal.timeout(timeoutMs),
    headers: {
      accept: 'application/json',
      'content-type': 'application/json',
      ...(userId ? { 'x-user-id': String(userId) } : {}),
      ...(secret ? { 'x-secret-key': String(secret) } : {}),
    },
    body: JSON.stringify(payload),
  });
  if (!response.ok) throw new Error(`Search service returned ${response.status}`);
  return response.json();
}

export async function searchGithubGateway(query, {
  endpoint = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL,
  limit = 10,
  timeoutMs = 12000,
  fetchImpl = fetch,
  secret = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET
    || process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const body = await postServiceJson(endpoint, '/v1/providers/github/search', {
    query,
    limit,
  }, {
    timeoutMs,
    fetchImpl,
    secret,
  });
  return (Array.isArray(body.results) ? body.results : [])
    .slice(0, limit)
    .map((item, index) => normalizeSearchResult({ ...item, source: 'github' }, index));
}

export async function searchResearchService(query, {
  endpoint,
  type = 'web',
  limit = 10,
  timeoutMs = 30000,
  fetchImpl = fetch,
  secret = process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const body = await postServiceJson(endpoint, '/v1/search', { query, type, limit }, {
    timeoutMs,
    fetchImpl,
    secret,
  });
  return (Array.isArray(body.results) ? body.results : [])
    .slice(0, limit)
    .map((item, index) => normalizeSearchResult({ ...item, source: item.source || 'research-service' }, index));
}

export async function startResearchTask(question, {
  endpoint,
  depth = 'standard',
  userId = null,
  llmProviderKeyId = '',
  llmModel = '',
  timeoutMs = 30000,
  fetchImpl = fetch,
  secret = process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const body = await postServiceJson(endpoint, '/v1/research', {
    question,
    depth,
    ...(userId ? { userId } : {}),
    ...(llmProviderKeyId && llmModel
      ? { llm_provider_key_id: llmProviderKeyId, llm_model: llmModel }
      : {}),
  }, { timeoutMs, fetchImpl, userId, secret });
  if (!body || typeof body !== 'object' || !body.task_id) throw new Error('Research service returned an invalid task');
  return {
    taskId: String(body.task_id),
    status: String(body.status ?? 'running'),
    service: String(body.service ?? 'research-service'),
  };
}

export async function getResearchTask(taskId, {
  endpoint,
  userId = null,
  timeoutMs = 30000,
  fetchImpl = fetch,
  secret = process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), {
    signal: AbortSignal.timeout(timeoutMs),
    headers: {
      accept: 'application/json',
      ...(userId ? { 'x-user-id': String(userId) } : {}),
      ...(secret ? { 'x-secret-key': String(secret) } : {}),
    },
  });
  if (response.status === 404) throw new Error('Research task was not found');
  if (!response.ok) throw new Error(`Search service returned ${response.status}`);
  return response.json();
}

export async function cancelResearchTask(taskId, {
  endpoint,
  userId = null,
  timeoutMs = 30000,
  fetchImpl = fetch,
  secret = process.env.TKMIND_DEEP_SEARCH_SECRET,
} = {}) {
  const response = await fetchImpl(serviceUrl(endpoint, `/v1/research/${encodeURIComponent(taskId)}`), {
    method: 'DELETE',
    signal: AbortSignal.timeout(timeoutMs),
    headers: {
      accept: 'application/json',
      ...(userId ? { 'x-user-id': String(userId) } : {}),
      ...(secret ? { 'x-secret-key': String(secret) } : {}),
    },
  });
  if (response.status === 404) throw new Error('Research task was not found');
  if (!response.ok) throw new Error(`Search service returned ${response.status}`);
  return response.json();
}