diff --git a/.env.example b/.env.example index 4b8df20..912b13a 100644 --- a/.env.example +++ b/.env.example @@ -270,6 +270,24 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind # Sandbox MCP 调用 Portal 内部生图入口；容器内通常使用 host.docker.internal。 # MINDSPACE_AGENT_API_BASE_URL=http://127.0.0.1:8081/api +# --------------------------------------------------------------------------- +# MindSearch / standalone Deep Search +# --------------------------------------------------------------------------- +# Production SearXNG on 103 is bound to the host at 127.0.0.1:20080. +# Goosed MCP endpoints are automatically rewritten to host.docker.internal. +# TKMIND_SEARCH_SEARXNG_URL=http://127.0.0.1:20080/search +# TKMIND_SEARCH_MCP_HOST_GATEWAY=host.docker.internal +# +# Deep Search runs as an independently released LaunchAgent on 127.0.0.1:20100. +# Its production environment is stored separately at: +# /Users/john/Project/deep-search-runtime/shared/.env +# TKMIND_DEEP_SEARCH_SECRET= +# TKMIND_SEARCH_GITHUB_GATEWAY_URL=http://127.0.0.1:20100 +# TKMIND_SEARCH_GITHUB_GATEWAY_SECRET=${TKMIND_DEEP_SEARCH_SECRET} +# +# GITHUB_TOKEN must only be set in the standalone Deep Search environment. +# Do not pass it to goosed MCP extension envs or commit it to Git. + # --------------------------------------------------------------------------- # Memind runtime profile（本地 vs 生产 必须区分） # 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。 diff --git a/.gitignore b/.gitignore index 5599414..629cc3f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ ops/node_modules/ dist/ ops/dist/ .runtime/ +.deep-search/ .env .env.local diff --git a/capabilities.mjs b/capabilities.mjs index c7576d6..3b26844 100644 --- a/capabilities.mjs +++ b/capabilities.mjs @@ -48,6 +48,23 @@ export function resolveMindSearchMcpServerPath(overridePath) { return path.join(path.dirname(fileURLToPath(import.meta.url)), 'tkmind-search-mcp.mjs'); } +export function resolveMindSearchMcpEndpoint( + endpoint, + { hostGateway = process.env.TKMIND_SEARCH_MCP_HOST_GATEWAY || 'host.docker.internal' } = {}, +) { + const source = String(endpoint ?? '').trim(); + if (!source) return source; + try { + const parsed = new URL(source); + if (LOOPBACK_PG_HOSTS.has(parsed.hostname)) { + parsed.hostname = String(hostGateway || 'host.docker.internal').trim(); + } + return parsed.toString(); + } catch { + return source; + } +} + export function resolveExcelMcpServerPath(overridePath) { const normalized = String(overridePath ?? '').trim(); if (normalized) return normalized; @@ -503,8 +520,21 @@ export function buildAgentExtensionPolicy( } const searchEnabled = capabilities.search_external && mindSearchConfig?.enabled && mindSearchConfig.mode !== 'off'; if (searchEnabled && (!policies || policies.network_egress !== 'deny')) { - const hasResearchService = mindSearchConfig.services?.some((service) => - service.enabled && service.id === mindSearchConfig.routes?.research && service.adapter === 'research-http'); + const mcpServices = (mindSearchConfig.services ?? []).map((service) => ({ + ...service, + endpoint: resolveMindSearchMcpEndpoint(service.endpoint), + })); + const researchService = mcpServices.find((service) => + service.id === mindSearchConfig.routes?.research && service.adapter === 'research-http'); + const deepSearchService = mcpServices.find((service) => + service.id === 'deep-search' && service.adapter === 'research-http'); + const hasResearchService = Boolean(researchService?.enabled); + const githubGatewayUrl = resolveMindSearchMcpEndpoint( + process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL + || deepSearchService?.endpoint + || researchService?.endpoint + || '', + ); extensions.push({ type: 'stdio', name: 'tkmind-search', @@ -519,14 +549,21 @@ export function buildAgentExtensionPolicy( TKMIND_SEARCH_PROVIDER_SEARXNG_ENABLED: mindSearchConfig.providers?.searxng ? '1' : '0', TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED: mindSearchConfig.providers?.github ? '1' : '0', TKMIND_SEARCH_READER_ENABLED: mindSearchConfig.providers?.reader ? '1' : '0', - TKMIND_SEARCH_SEARXNG_URL: mindSearchConfig.settings?.searxngEndpoint ?? '', + TKMIND_SEARCH_SEARXNG_URL: resolveMindSearchMcpEndpoint( + mindSearchConfig.settings?.searxngEndpoint ?? '', + ), TKMIND_SEARCH_MAX_RESULTS: String(mindSearchConfig.settings?.maxResults ?? 10), TKMIND_SEARCH_TIMEOUT_MS: String(mindSearchConfig.settings?.timeoutMs ?? 8000), TKMIND_SEARCH_READER_MAX_CHARS: String(mindSearchConfig.settings?.readerMaxChars ?? 12000), - TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mindSearchConfig.services ?? []), + TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mcpServices), TKMIND_SEARCH_ROUTES_JSON: JSON.stringify(mindSearchConfig.routes ?? {}), TKMIND_SEARCH_USER_ID: userId ? String(userId) : '', TKMIND_DEEP_SEARCH_SECRET: process.env.TKMIND_DEEP_SEARCH_SECRET ?? '', + TKMIND_SEARCH_GITHUB_GATEWAY_URL: githubGatewayUrl, + TKMIND_SEARCH_GITHUB_GATEWAY_SECRET: + process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET + ?? process.env.TKMIND_DEEP_SEARCH_SECRET + ?? '', }, available_tools: [ 'tkmind_search', diff --git a/capabilities.test.mjs b/capabilities.test.mjs index 84bf9cd..be36471 100644 --- a/capabilities.test.mjs +++ b/capabilities.test.mjs @@ -8,6 +8,7 @@ import { clampUserCapabilities, DEFAULT_USER_CAPABILITIES, normalizeCapabilityPatch, + resolveMindSearchMcpEndpoint, resolveSandboxMcpNodeExecPath, resolveSandboxMcpServerPath, resolveSandboxMcpUserDataPgUrl, @@ -53,6 +54,17 @@ test('resolveSandboxMcpUserDataPgUrl preserves native URLs and honors an explici ); }); +test('resolveMindSearchMcpEndpoint rewrites host loopback for goosed containers', () => { + assert.equal( + resolveMindSearchMcpEndpoint('http://127.0.0.1:20080/search'), + 'http://host.docker.internal:20080/search', + ); + assert.equal( + resolveMindSearchMcpEndpoint('https://search.internal/query'), + 'https://search.internal/query', + ); +}); + test('resolveSandboxMcpNodeExecPath honors container-path override without host fs checks', () => { assert.equal(resolveSandboxMcpNodeExecPath('/usr/local/bin/node'), '/usr/local/bin/node'); assert.equal(resolveSandboxMcpNodeExecPath(''), process.execPath); diff --git a/deep-search-server.mjs b/deep-search-server.mjs index 9cb2abd..b0004fa 100644 --- a/deep-search-server.mjs +++ b/deep-search-server.mjs @@ -6,6 +6,7 @@ import { createPortalGatewayResearchLlmResolver, } from './deep-search-engine.mjs'; import { createDeepSearchStore } from './deep-search-store.mjs'; +import { searchGithubCode } from './mindsearch-providers.mjs'; function sendJson(res, status, body) { const payload = JSON.stringify(body); @@ -63,19 +64,50 @@ function canAccessTask(task, requestUserId) { export function createDeepSearchHttpServer({ engine, secret = process.env.TKMIND_DEEP_SEARCH_SECRET || '', + githubToken = process.env.GITHUB_TOKEN || '', + githubSearch = searchGithubCode, logger = console, } = {}) { if (!engine) throw new Error('Deep Search engine is required'); return http.createServer(async (req, res) => { const url = new URL(req.url ?? '/', 'http://deep-search.local'); if (url.pathname === '/health' && req.method === 'GET') { - return sendJson(res, 200, engine.getHealth()); + return sendJson(res, 200, { + ...engine.getHealth(), + providers: { + github: { configured: Boolean(githubToken) }, + }, + }); } if (secret && req.headers['x-secret-key'] !== secret) { return sendJson(res, 401, { code: 'UNAUTHORIZED', message: 'Invalid service secret' }); } const requestUserId = String(req.headers['x-user-id'] ?? '').trim(); try { + if (url.pathname === '/v1/providers/github/search' && req.method === 'POST') { + if (!githubToken) { + const error = new Error('GitHub search token is not configured'); + error.status = 503; + throw error; + } + const input = await readJson(req, { maxBytes: 32_000 }); + const query = String(input.query ?? '').trim(); + if (!query || query.length > 500) { + const error = new Error('query must be 1-500 characters'); + error.status = 400; + throw error; + } + const limit = Math.max(1, Math.min(20, Number(input.limit ?? 10) || 10)); + const results = await githubSearch(query, { + limit, + token: githubToken, + }); + return sendJson(res, 200, { + query, + results, + service: 'tkmind-deep-search', + }); + } if (url.pathname === '/v1/search' && req.method === 'POST') { const input = await readJson(req); const results = await engine.search(input); diff --git a/deep-search.test.mjs b/deep-search.test.mjs index d4e81b3..eadb154 100644 --- a/deep-search.test.mjs +++ b/deep-search.test.mjs @@ -281,7 +281,23 @@ test('Deep Search supports cancellation while a provider is running', async () = test('Deep Search HTTP service exposes health, async start, status, search, and auth', async (t) => { const { engine, store } = createMockEngine({ id: 'task-http' }); - const server = createDeepSearchHttpServer({ engine, secret: 'local-secret', logger: { warn() {} } }); + let githubRequest; + const server = createDeepSearchHttpServer({ + engine, + secret: 'local-secret', + githubToken: 'github-secret-token', + githubSearch: async (query, options) => { + githubRequest = { query, options }; + return [{ + title: 'tkmind/code:server.mjs', + url: 'https://github.com/tkmind/code/blob/main/server.mjs', + snippet: 'server', + source: 'github', + rank: 1, + }]; + }, + logger: { warn() {} }, + }); await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); t.after(async () => { await new Promise((resolve) => server.close(resolve)); @@ -291,8 +307,17 @@ test('Deep Search HTTP service exposes health, async start, status, search, and const base = `http://127.0.0.1:${address.port}`; const health = await fetch(`${base}/health`).then((response) => response.json()); assert.equal(health.ok, true); + assert.equal(health.providers.github.configured, true); assert.equal((await fetch(`${base}/v1/research`)).status, 401); const headers = { 'content-type': 'application/json', 'x-secret-key': 'local-secret' }; + const github = await fetch(`${base}/v1/providers/github/search`, { + method: 'POST', + headers, + body: JSON.stringify({ query: 'repo:tkmind/code server', limit: 3 }), + }).then((response) => response.json()); + assert.equal(github.results.length, 1); + assert.equal(githubRequest.options.token, 'github-secret-token'); + assert.equal(githubRequest.options.limit, 3); const search = await fetch(`${base}/v1/search`, { method: 'POST', headers, diff --git a/docs/deep-search-service.md b/docs/deep-search-service.md index 5059a22..ea062b8 100644 --- a/docs/deep-search-service.md +++ b/docs/deep-search-service.md @@ -30,6 +30,11 @@ npm run dev:deep-search The server listens on `127.0.0.1:20100` by default. +On production host `103`, SearXNG is exposed at +`http://127.0.0.1:20080/search`. Goosed containers reach host services through +`host.docker.internal`; Portal rewrites loopback MindSearch service endpoints +only when constructing the MCP extension environment. + ## API | Method | Path | Purpose | @@ -73,6 +78,7 @@ Start response: | `TKMIND_DEEP_SEARCH_SECRET` | empty | Optional `X-Secret-Key` required by non-health routes | | `TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL` | `http://127.0.0.1:8081/api/internal/deep-search/llm` | Portal LLM gateway | | `TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET` | `TKMIND_DEEP_SEARCH_SECRET` | Portal gateway bearer secret | +| `GITHUB_TOKEN` | empty | GitHub code-search token, stored only in the standalone service environment | ## Isolation and safety @@ -84,6 +90,40 @@ Start response: - Source reading rejects credentials, loopback, link-local, private, multicast, and private-DNS destinations. - Redirect targets are resolved and validated again. - Request bodies, result counts, source content, timeouts, and task depths are bounded. +- GitHub Code Search is proxied by Deep Search so the provider token is never + copied into goosed extension environments. + +## Production runtime on 103 + +Deep Search is released independently from Portal and `memindadm`: + +```bash +npm run build:deep-search-runtime +CONFIRMED_CI_SHA="$(git rev-parse HEAD)" \ + bash scripts/release-deep-search-runtime-prod.sh +``` + +The production layout is: + +```text +/Users/john/Project/deep-search-runtime/ +├── current -> releases// +├── releases// +├── shared/.env +├── data/research.sqlite +├── logs/deep-search.log +└── backups// +``` + +The release workflow requires a clean `main` that exactly matches +`origin/main`, a matching confirmed CI SHA, artifact checksum verification, +SQLite online backup, LaunchAgent rollback, host health/search probes, and a +Goose-container reachability probe. It never releases or restarts Portal or +`memindadm`. + +The installer creates a random service secret on first install. Add a newly +issued, least-privilege GitHub token to `shared/.env` only after revoking any +token that has appeared in chat or logs. ## MindSearch registration diff --git a/mindsearch-providers.mjs b/mindsearch-providers.mjs index d21929f..b669d39 100644 --- a/mindsearch-providers.mjs +++ b/mindsearch-providers.mjs @@ -68,6 +68,27 @@ async function postServiceJson(endpoint, path, payload, { return response.json(); } +export async function searchGithubGateway(query, { + endpoint = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL, + limit = 10, + timeoutMs = 12000, + fetchImpl = fetch, + secret = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET + || process.env.TKMIND_DEEP_SEARCH_SECRET, +} = {}) { + const body = await postServiceJson(endpoint, '/v1/providers/github/search', { + query, + limit, + }, { + timeoutMs, + fetchImpl, + secret, + }); + return (Array.isArray(body.results) ? body.results : []) + .slice(0, limit) + .map((item, index) => normalizeSearchResult({ ...item, source: 'github' }, index)); +} + export async function searchResearchService(query, { endpoint, type = 'web', diff --git a/mindsearch.test.mjs b/mindsearch.test.mjs index 42cc1cb..3f20559 100644 --- a/mindsearch.test.mjs +++ b/mindsearch.test.mjs @@ -7,11 +7,15 @@ import { probeMindSearchService, resolveMindSearchService, } from './mindsearch-config.mjs'; -import { buildAgentExtensionPolicy, DEFAULT_USER_CAPABILITIES } from './capabilities.mjs'; +import { + buildAgentExtensionPolicy, + DEFAULT_USER_CAPABILITIES, +} from './capabilities.mjs'; import { cancelResearchTask, getResearchTask, searchGithubCode, + searchGithubGateway, searchResearchService, searchSearxng, startResearchTask, @@ -102,12 +106,40 @@ test('MindSearch never changes legacy web extension and is gated by capability/c enabled: true, mode: 'assist', providers: {}, - services: [{ id: 'deep-search', enabled: true, adapter: 'research-http' }], + settings: { searxngEndpoint: 'http://127.0.0.1:20080/search' }, + services: [ + { + id: 'searxng', + enabled: true, + adapter: 'searxng', + endpoint: 'http://127.0.0.1:20080/search', + }, + { + id: 'deep-search', + enabled: true, + adapter: 'research-http', + endpoint: 'http://127.0.0.1:20100', + }, + ], routes: { research: 'deep-search' }, }, }); const extension = policy.extensionOverrides.find((ext) => ext.name === 'tkmind-search'); assert.equal(extension.envs.TKMIND_SEARCH_USER_ID, 'user-123'); + assert.equal( + extension.envs.TKMIND_SEARCH_SEARXNG_URL, + 'http://host.docker.internal:20080/search', + ); + assert.equal( + extension.envs.TKMIND_SEARCH_GITHUB_GATEWAY_URL, + 'http://host.docker.internal:20100/', + ); + assert.equal(extension.envs.GITHUB_TOKEN, undefined); + const mcpServices = JSON.parse(extension.envs.TKMIND_SEARCH_SERVICES_JSON); + assert.equal( + mcpServices.find((service) => service.id === 'deep-search').endpoint, + 'http://host.docker.internal:20100/', + ); assert.ok(extension.available_tools.includes('tkmind_research_status')); assert.ok(extension.available_tools.includes('tkmind_research_cancel')); }); @@ -125,6 +157,31 @@ test('GitHub code adapter sends bounded queries and normalizes results', async ( assert.equal(result[0].source, 'github'); }); +test('GitHub gateway keeps the provider token outside the MCP process', async () => { + let requested; + const result = await searchGithubGateway('repo:openai goose', { + endpoint: 'http://deep-search.local:20100', + secret: 'gateway-secret', + fetchImpl: async (url, options) => { + requested = { url: String(url), options }; + return { + ok: true, + json: async () => ({ + results: [{ + title: 'openai/goose:src/goose.rs', + url: 'https://github.com/openai/goose/blob/main/src/goose.rs', + snippet: 'agent', + }], + }), + }; + }, + }); + assert.equal(requested.url, 'http://deep-search.local:20100/v1/providers/github/search'); + assert.equal(requested.options.headers['x-secret-key'], 'gateway-secret'); + assert.equal(requested.options.headers.authorization, undefined); + assert.equal(result[0].source, 'github'); +}); + test('Research service adapter uses the stable HTTP contract', async () => { const requests = []; const fetchImpl = async (url, options) => { diff --git a/package.json b/package.json index 738eb46..a91fff2 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "backfill:plaza-public-urls": "node scripts/backfill-plaza-public-urls.mjs", "enrich:plaza": "node scripts/enrich-plaza-posts.mjs", "build": "vite build", + "build:deep-search-runtime": "node scripts/build-deep-search-runtime.mjs", "build:portal-runtime": "node scripts/build-portal-runtime.mjs", "build:imgproxy-runtime": "bash scripts/build-imgproxy-runtime-image.sh", "build:mindspace-service-runtime": "node scripts/build-mindspace-service-runtime.mjs", diff --git a/scripts/build-deep-search-runtime.mjs b/scripts/build-deep-search-runtime.mjs new file mode 100644 index 0000000..829b953 --- /dev/null +++ b/scripts/build-deep-search-runtime.mjs @@ -0,0 +1,75 @@ +import { spawn } from 'node:child_process'; +import { + chmod, + copyFile, + mkdir, + rm, + writeFile, +} from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const runtimeRoot = path.join(root, '.runtime', 'deep-search'); +const esbuildBin = path.join(root, 'node_modules', '.bin', 'esbuild'); +const runtimeNodeTarget = process.env.DEEP_SEARCH_RUNTIME_NODE_TARGET || 'node24'; + +function run(command, args) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd: root, + stdio: 'inherit', + env: process.env, + }); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (code === 0) return resolve(); + reject(new Error(`${command} exited with ${code ?? signal}`)); + }); + }); +} + +await rm(runtimeRoot, { recursive: true, force: true }); +await mkdir(path.join(runtimeRoot, 'scripts'), { recursive: true }); + +await run(esbuildBin, [ + 'deep-search-server.mjs', + '--bundle', + '--platform=node', + '--format=esm', + `--target=${runtimeNodeTarget}`, + '--outfile=.runtime/deep-search/deep-search-server.mjs', + '--banner:js=import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', +]); + +for (const script of [ + 'run-deep-search-prod.sh', + 'install-deep-search-runtime-prod.sh', +]) { + const source = path.join(root, 'scripts', script); + const target = path.join(runtimeRoot, 'scripts', script); + await copyFile(source, target); + await chmod(target, 0o755); +} + +await writeFile( + path.join(runtimeRoot, 'RUNBOOK.txt'), + [ + 'TKMind Deep Search standalone runtime', + '', + 'Production base: /Users/john/Project/deep-search-runtime', + 'Persistent env: shared/.env', + 'Persistent data: data/research.sqlite', + 'Release code: releases//', + 'Active release: current -> releases//', + 'LaunchAgent: cn.tkmind.memind-deep-search', + 'Health: http://127.0.0.1:20100/health', + 'SearXNG: http://127.0.0.1:20080/search', + '', + 'The GitHub token stays in shared/.env and is never passed to goosed MCP processes.', + 'Portal and memindadm releases are intentionally separate.', + '', + ].join('\n'), +); + +console.log(`Deep Search runtime built at ${runtimeRoot}`); diff --git a/scripts/deep-search-runtime-package.test.mjs b/scripts/deep-search-runtime-package.test.mjs new file mode 100644 index 0000000..fbd50ad --- /dev/null +++ b/scripts/deep-search-runtime-package.test.mjs @@ -0,0 +1,31 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const read = (relativePath) => readFileSync(path.join(root, relativePath), 'utf8'); + +test('Deep Search production runtime is isolated from Portal releases', () => { + const builder = read('scripts/build-deep-search-runtime.mjs'); + assert.match(builder, /deep-search-server\.mjs/); + assert.match(builder, /\.runtime', 'deep-search/); + + const runner = read('scripts/run-deep-search-prod.sh'); + assert.match(runner, /shared\/\.env/); + assert.match(runner, /data\/research\.sqlite/); + assert.match(runner, /127\.0\.0\.1:20080\/search/); + + const installer = read('scripts/install-deep-search-runtime-prod.sh'); + assert.match(installer, /cn\.tkmind\.memind-deep-search/); + assert.match(installer, /chmod 600/); + assert.match(installer, /openssl rand -hex 32/); + + const release = read('scripts/release-deep-search-runtime-prod.sh'); + assert.match(release, /branch.*main/); + assert.match(release, /CONFIRMED_CI_SHA/); + assert.match(release, /shasum -a 256/); + assert.match(release, /rollback/); + assert.doesNotMatch(release, /release-portal-runtime-prod/); +}); diff --git a/scripts/install-deep-search-runtime-prod.sh b/scripts/install-deep-search-runtime-prod.sh new file mode 100644 index 0000000..9368483 --- /dev/null +++ b/scripts/install-deep-search-runtime-prod.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Runs on 103 from a verified Deep Search release directory. +set -euo pipefail + +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}" +RELEASE_DIR="${DEEP_SEARCH_RELEASE_DIR:?DEEP_SEARCH_RELEASE_DIR is required}" +ENV_DIR="${RUNTIME_BASE}/shared" +ENV_FILE="${ENV_DIR}/.env" +DATA_DIR="${RUNTIME_BASE}/data" +LOG_DIR="${RUNTIME_BASE}/logs" +PLIST="${HOME}/Library/LaunchAgents/cn.tkmind.memind-deep-search.plist" +LABEL="cn.tkmind.memind-deep-search" +GUI="gui/$(id -u)" +NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}" + +[[ -x "${NODE_BIN}" ]] || { + echo "missing Node.js runtime: ${NODE_BIN}" >&2 + exit 1 +} +[[ -f "${RELEASE_DIR}/deep-search-server.mjs" ]] || { + echo "missing Deep Search server in ${RELEASE_DIR}" >&2 + exit 1 +} + +mkdir -p "${ENV_DIR}" "${DATA_DIR}" "${LOG_DIR}" "$(dirname "${PLIST}")" +touch "${ENV_FILE}" +chmod 600 "${ENV_FILE}" + +ensure_env() { + local key="$1" + local value="$2" + if ! grep -qE "^${key}=" "${ENV_FILE}"; then + printf '%s=%s\n' "${key}" "${value}" >> "${ENV_FILE}" + fi +} + +if ! grep -qE '^TKMIND_DEEP_SEARCH_SECRET=.+$' "${ENV_FILE}"; then + secret="$(openssl rand -hex 32)" + ensure_env TKMIND_DEEP_SEARCH_SECRET "${secret}" +fi +ensure_env TKMIND_DEEP_SEARCH_HOST "127.0.0.1" +ensure_env TKMIND_DEEP_SEARCH_PORT "20100" +ensure_env TKMIND_DEEP_SEARCH_DB "${DATA_DIR}/research.sqlite" +ensure_env TKMIND_DEEP_SEARCH_SEARXNG_URL "http://127.0.0.1:20080/search" +ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL "http://127.0.0.1:8081/api/internal/deep-search/llm" +ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET '${TKMIND_DEEP_SEARCH_SECRET}' + +ln -sfn "${RELEASE_DIR}" "${RUNTIME_BASE}/current" + +cat > "${PLIST}" < + + +Label${LABEL} +ProgramArguments${RUNTIME_BASE}/current/scripts/run-deep-search-prod.sh +WorkingDirectory${RUNTIME_BASE}/current +EnvironmentVariables + DEEP_SEARCH_RUNTIME_BASE${RUNTIME_BASE} + NODE_BIN${NODE_BIN} + +RunAtLoad +KeepAlive +ProcessTypeBackground +ThrottleInterval10 +StandardOutPath${LOG_DIR}/deep-search.log +StandardErrorPath${LOG_DIR}/deep-search.log + +EOF + +plutil -lint "${PLIST}" >/dev/null +launchctl bootout "${GUI}/${LABEL}" >/dev/null 2>&1 || true +launchctl bootstrap "${GUI}" "${PLIST}" +launchctl enable "${GUI}/${LABEL}" +launchctl kickstart -k "${GUI}/${LABEL}" + +for _ in $(seq 1 30); do + if curl -fsS --max-time 2 "http://127.0.0.1:20100/health" >/dev/null; then + exit 0 + fi + sleep 1 +done + +echo "Deep Search health check timed out" >&2 +exit 1 diff --git a/scripts/release-deep-search-runtime-prod.sh b/scripts/release-deep-search-runtime-prod.sh new file mode 100644 index 0000000..142d2cb --- /dev/null +++ b/scripts/release-deep-search-runtime-prod.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash +# Publishes only the independently versioned Deep Search runtime to 103. +# Portal and memindadm are not modified by this workflow. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOST="${STUDIO_HOST:-58.38.22.103}" +REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}" +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-${REMOTE_ROOT}/deep-search-runtime}" +INCOMING="${REMOTE_ROOT}/incoming/deep-search-runtime" +RELEASE_ID="$(date +%Y%m%d-%H%M%S)-$(git -C "${ROOT}" rev-parse --short HEAD)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/deep-search-runtime-release.XXXXXX")" +DRY_RUN=0 +AUTO_YES=0 + +cleanup() { + rm -rf "${TMP_DIR}" +} +trap cleanup EXIT + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) DRY_RUN=1 ;; + --yes) AUTO_YES=1 ;; + -h|--help) + cat <<'EOF' +Usage: bash scripts/release-deep-search-runtime-prod.sh [--dry-run] [--yes] + +Required production gates: + - current branch is main + - HEAD exactly matches origin/main + - worktree is clean + - CONFIRMED_CI_SHA equals the full current HEAD + +This installs only Deep Search on 103. It does not release Portal or memindadm. +EOF + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + exit 1 + ;; + esac + shift +done + +git -C "${ROOT}" fetch origin main --quiet +branch="$(git -C "${ROOT}" branch --show-current)" +head_sha="$(git -C "${ROOT}" rev-parse HEAD)" +origin_sha="$(git -C "${ROOT}" rev-parse origin/main)" +[[ "${branch}" == "main" ]] || { + echo "Deep Search production release must run from main" >&2 + exit 1 +} +[[ "${head_sha}" == "${origin_sha}" ]] || { + echo "main must exactly match origin/main" >&2 + exit 1 +} +[[ -z "$(git -C "${ROOT}" status --porcelain=v1 --untracked-files=all)" ]] || { + echo "worktree must be clean" >&2 + exit 1 +} +[[ "${CONFIRMED_CI_SHA:-}" == "${head_sha}" ]] || { + echo "CONFIRMED_CI_SHA must equal the verified main HEAD: ${head_sha}" >&2 + exit 1 +} +ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh" + +( + cd "${ROOT}" + npm run test:deep-search + node --test capabilities.test.mjs scripts/deep-search-runtime-package.test.mjs + npm run build:deep-search-runtime +) + +RUNTIME_DIR="${ROOT}/.runtime/deep-search" +for file in \ + deep-search-server.mjs \ + scripts/run-deep-search-prod.sh \ + scripts/install-deep-search-runtime-prod.sh \ + RUNBOOK.txt; do + [[ -f "${RUNTIME_DIR}/${file}" ]] || { + echo "missing runtime artifact: ${file}" >&2 + exit 1 + } +done + +MANIFEST="${RUNTIME_DIR}/release-manifest.txt" +{ + echo "release_id=${RELEASE_ID}" + echo "git_head=${head_sha}" + echo "git_branch=${branch}" + echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')" + echo "artifact=.runtime/deep-search" + echo "persistent_env=${RUNTIME_BASE}/shared/.env" + echo "persistent_db=${RUNTIME_BASE}/data/research.sqlite" +} > "${MANIFEST}" + +ARCHIVE="${TMP_DIR}/deep-search-runtime-${RELEASE_ID}.tar.gz" +SHA_FILE="${ARCHIVE}.sha256" +tar -C "${RUNTIME_DIR}" -czf "${ARCHIVE}" . +(cd "${TMP_DIR}" && shasum -a 256 "$(basename "${ARCHIVE}")" > "$(basename "${SHA_FILE}")") + +if [[ "${DRY_RUN}" -eq 1 ]]; then + shasum -a 256 -c "${SHA_FILE}" + echo "Deep Search dry-run artifact verified: ${ARCHIVE}" + exit 0 +fi + +if [[ "${AUTO_YES}" -ne 1 ]]; then + echo "Target: ${HOST}:${RUNTIME_BASE}" + echo "Release: ${RELEASE_ID}" + read -r -p "Install standalone Deep Search on 103? [y/N] " confirm /dev/null || true)" +database_path="${RUNTIME_BASE}/data/research.sqlite" + +mkdir -p "${release_dir}" "${backup_dir}" +cp "${plist}" "${backup_dir}/launchagent.plist" 2>/dev/null || true +if [[ -f "${database_path}" ]]; then + sqlite3 "${database_path}" ".timeout 10000" \ + ".backup '${backup_dir}/research.sqlite'" >/dev/null +fi +cd "${INCOMING}" +shasum -a 256 -c "$(basename "${sha_file}")" +tar -xzf "${archive}" -C "${release_dir}" + +rollback() { + if [[ -n "${previous_target}" && -d "${previous_target}" ]]; then + ln -sfn "${previous_target}" "${current_link}" + else + rm -f "${current_link}" + fi + if [[ -f "${backup_dir}/launchagent.plist" ]]; then + cp "${backup_dir}/launchagent.plist" "${plist}" + launchctl bootout "${gui}/${label}" >/dev/null 2>&1 || true + launchctl bootstrap "${gui}" "${plist}" >/dev/null 2>&1 || true + launchctl kickstart -k "${gui}/${label}" >/dev/null 2>&1 || true + fi +} +trap rollback ERR + +DEEP_SEARCH_RUNTIME_BASE="${RUNTIME_BASE}" \ +DEEP_SEARCH_RELEASE_DIR="${release_dir}" \ + bash "${release_dir}/scripts/install-deep-search-runtime-prod.sh" + +set -a +# shellcheck disable=SC1090 +source "${RUNTIME_BASE}/shared/.env" +set +a + +curl -fsS --max-time 5 "http://127.0.0.1:20100/health" >/dev/null +curl -fsS --max-time 20 \ + -H "X-Secret-Key: ${TKMIND_DEEP_SEARCH_SECRET}" \ + -H "Content-Type: application/json" \ + -d '{"query":"OpenAI","limit":3}' \ + "http://127.0.0.1:20100/v1/search" \ + | grep -q '"results"' + +docker_bin="" +for candidate in /usr/local/bin/docker /opt/homebrew/bin/docker; do + if [[ -x "${candidate}" ]]; then + docker_bin="${candidate}" + break + fi +done +if [[ -n "${docker_bin}" ]]; then + goosed_container="$("${docker_bin}" ps --format '{{.Names}}' | grep -E '^goosed-prod-' | head -1)" + if [[ -n "${goosed_container}" ]]; then + "${docker_bin}" exec "${goosed_container}" sh -lc \ + 'curl -fsS --max-time 5 http://host.docker.internal:20100/health >/dev/null' + fi +fi + +trap - ERR +printf '%s\n' "${RELEASE_ID}" > "${RUNTIME_BASE}/current-release.txt" +REMOTE + +echo "Deep Search installed and verified on 103: ${RELEASE_ID}" diff --git a/scripts/run-deep-search-prod.sh b/scripts/run-deep-search-prod.sh new file mode 100644 index 0000000..842538e --- /dev/null +++ b/scripts/run-deep-search-prod.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUNTIME_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}" +ENV_FILE="${DEEP_SEARCH_ENV_FILE:-${RUNTIME_BASE}/shared/.env}" + +if [[ ! -f "${ENV_FILE}" ]]; then + echo "[deep-search-start] missing environment file: ${ENV_FILE}" >&2 + exit 1 +fi + +set -a +# shellcheck disable=SC1090 +source "${ENV_FILE}" +set +a + +: "${TKMIND_DEEP_SEARCH_SECRET:?missing TKMIND_DEEP_SEARCH_SECRET}" + +export NODE_ENV=production +export TKMIND_DEEP_SEARCH_HOST="${TKMIND_DEEP_SEARCH_HOST:-127.0.0.1}" +export TKMIND_DEEP_SEARCH_PORT="${TKMIND_DEEP_SEARCH_PORT:-20100}" +export TKMIND_DEEP_SEARCH_DB="${TKMIND_DEEP_SEARCH_DB:-${RUNTIME_BASE}/data/research.sqlite}" +export TKMIND_DEEP_SEARCH_SEARXNG_URL="${TKMIND_DEEP_SEARCH_SEARXNG_URL:-http://127.0.0.1:20080/search}" +export TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL:-http://127.0.0.1:8081/api/internal/deep-search/llm}" +export TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET:-${TKMIND_DEEP_SEARCH_SECRET}}" + +mkdir -p "$(dirname "${TKMIND_DEEP_SEARCH_DB}")" "${RUNTIME_BASE}/logs" + +NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}" +if [[ ! -x "${NODE_BIN}" ]]; then + NODE_BIN="$(command -v node)" +fi + +exec "${NODE_BIN}" "${RUNTIME_ROOT}/deep-search-server.mjs" diff --git a/tkmind-search-mcp.mjs b/tkmind-search-mcp.mjs index da459cd..52af9ad 100644 --- a/tkmind-search-mcp.mjs +++ b/tkmind-search-mcp.mjs @@ -6,6 +6,7 @@ import { getResearchTask, readSafeUrl, searchGithubCode, + searchGithubGateway, searchResearchService, searchSearxng, startResearchTask, @@ -86,7 +87,13 @@ function handle(message) { const route = checked.value.type === 'code' ? 'code' : checked.value.type === 'news' ? 'news' : 'web'; const service = resolveMindSearchService(config, route); if (service?.adapter === 'github') { - return searchGithubCode(checked.value.query, { limit: checked.value.limit, endpoint: service.endpoint || undefined }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); + const operation = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL + ? searchGithubGateway + : searchGithubCode; + return operation(checked.value.query, { + limit: checked.value.limit, + endpoint: process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL || service.endpoint || undefined, + }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message)); } if (service?.adapter === 'searxng') { return searchSearxng(checked.value.query, { limit: checked.value.limit, endpoint: service.endpoint }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message));