feat: prepare deep search production runtime
Memind CI / Test, build, and release guards (pull_request) Successful in 4m4s
Memind CI / Test, build, and release guards (pull_request) Successful in 4m4s
This commit is contained in:
@@ -270,6 +270,24 @@ VITE_TKMIND_WORKING_DIR=/Users/john/PycharmProjects/tkmind
|
||||
# Sandbox MCP 调用 Portal 内部生图入口;容器内通常使用 host.docker.internal。
|
||||
# MINDSPACE_AGENT_API_BASE_URL=http://127.0.0.1:8081/api
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MindSearch / standalone Deep Search
|
||||
# ---------------------------------------------------------------------------
|
||||
# Production SearXNG on 103 is bound to the host at 127.0.0.1:20080.
|
||||
# Goosed MCP endpoints are automatically rewritten to host.docker.internal.
|
||||
# TKMIND_SEARCH_SEARXNG_URL=http://127.0.0.1:20080/search
|
||||
# TKMIND_SEARCH_MCP_HOST_GATEWAY=host.docker.internal
|
||||
#
|
||||
# Deep Search runs as an independently released LaunchAgent on 127.0.0.1:20100.
|
||||
# Its production environment is stored separately at:
|
||||
# /Users/john/Project/deep-search-runtime/shared/.env
|
||||
# TKMIND_DEEP_SEARCH_SECRET=
|
||||
# TKMIND_SEARCH_GITHUB_GATEWAY_URL=http://127.0.0.1:20100
|
||||
# TKMIND_SEARCH_GITHUB_GATEWAY_SECRET=${TKMIND_DEEP_SEARCH_SECRET}
|
||||
#
|
||||
# GITHUB_TOKEN must only be set in the standalone Deep Search environment.
|
||||
# Do not pass it to goosed MCP extension envs or commit it to Git.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Memind runtime profile(本地 vs 生产 必须区分)
|
||||
# 由 scripts/memind-runtime-profile.mjs 在 pnpm dev / server.mjs 启动时应用。
|
||||
|
||||
@@ -3,6 +3,7 @@ ops/node_modules/
|
||||
dist/
|
||||
ops/dist/
|
||||
.runtime/
|
||||
.deep-search/
|
||||
|
||||
.env
|
||||
.env.local
|
||||
|
||||
+41
-4
@@ -48,6 +48,23 @@ export function resolveMindSearchMcpServerPath(overridePath) {
|
||||
return path.join(path.dirname(fileURLToPath(import.meta.url)), 'tkmind-search-mcp.mjs');
|
||||
}
|
||||
|
||||
export function resolveMindSearchMcpEndpoint(
|
||||
endpoint,
|
||||
{ hostGateway = process.env.TKMIND_SEARCH_MCP_HOST_GATEWAY || 'host.docker.internal' } = {},
|
||||
) {
|
||||
const source = String(endpoint ?? '').trim();
|
||||
if (!source) return source;
|
||||
try {
|
||||
const parsed = new URL(source);
|
||||
if (LOOPBACK_PG_HOSTS.has(parsed.hostname)) {
|
||||
parsed.hostname = String(hostGateway || 'host.docker.internal').trim();
|
||||
}
|
||||
return parsed.toString();
|
||||
} catch {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveExcelMcpServerPath(overridePath) {
|
||||
const normalized = String(overridePath ?? '').trim();
|
||||
if (normalized) return normalized;
|
||||
@@ -503,8 +520,21 @@ export function buildAgentExtensionPolicy(
|
||||
}
|
||||
const searchEnabled = capabilities.search_external && mindSearchConfig?.enabled && mindSearchConfig.mode !== 'off';
|
||||
if (searchEnabled && (!policies || policies.network_egress !== 'deny')) {
|
||||
const hasResearchService = mindSearchConfig.services?.some((service) =>
|
||||
service.enabled && service.id === mindSearchConfig.routes?.research && service.adapter === 'research-http');
|
||||
const mcpServices = (mindSearchConfig.services ?? []).map((service) => ({
|
||||
...service,
|
||||
endpoint: resolveMindSearchMcpEndpoint(service.endpoint),
|
||||
}));
|
||||
const researchService = mcpServices.find((service) =>
|
||||
service.id === mindSearchConfig.routes?.research && service.adapter === 'research-http');
|
||||
const deepSearchService = mcpServices.find((service) =>
|
||||
service.id === 'deep-search' && service.adapter === 'research-http');
|
||||
const hasResearchService = Boolean(researchService?.enabled);
|
||||
const githubGatewayUrl = resolveMindSearchMcpEndpoint(
|
||||
process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL
|
||||
|| deepSearchService?.endpoint
|
||||
|| researchService?.endpoint
|
||||
|| '',
|
||||
);
|
||||
extensions.push({
|
||||
type: 'stdio',
|
||||
name: 'tkmind-search',
|
||||
@@ -519,14 +549,21 @@ export function buildAgentExtensionPolicy(
|
||||
TKMIND_SEARCH_PROVIDER_SEARXNG_ENABLED: mindSearchConfig.providers?.searxng ? '1' : '0',
|
||||
TKMIND_SEARCH_PROVIDER_GITHUB_ENABLED: mindSearchConfig.providers?.github ? '1' : '0',
|
||||
TKMIND_SEARCH_READER_ENABLED: mindSearchConfig.providers?.reader ? '1' : '0',
|
||||
TKMIND_SEARCH_SEARXNG_URL: mindSearchConfig.settings?.searxngEndpoint ?? '',
|
||||
TKMIND_SEARCH_SEARXNG_URL: resolveMindSearchMcpEndpoint(
|
||||
mindSearchConfig.settings?.searxngEndpoint ?? '',
|
||||
),
|
||||
TKMIND_SEARCH_MAX_RESULTS: String(mindSearchConfig.settings?.maxResults ?? 10),
|
||||
TKMIND_SEARCH_TIMEOUT_MS: String(mindSearchConfig.settings?.timeoutMs ?? 8000),
|
||||
TKMIND_SEARCH_READER_MAX_CHARS: String(mindSearchConfig.settings?.readerMaxChars ?? 12000),
|
||||
TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mindSearchConfig.services ?? []),
|
||||
TKMIND_SEARCH_SERVICES_JSON: JSON.stringify(mcpServices),
|
||||
TKMIND_SEARCH_ROUTES_JSON: JSON.stringify(mindSearchConfig.routes ?? {}),
|
||||
TKMIND_SEARCH_USER_ID: userId ? String(userId) : '',
|
||||
TKMIND_DEEP_SEARCH_SECRET: process.env.TKMIND_DEEP_SEARCH_SECRET ?? '',
|
||||
TKMIND_SEARCH_GITHUB_GATEWAY_URL: githubGatewayUrl,
|
||||
TKMIND_SEARCH_GITHUB_GATEWAY_SECRET:
|
||||
process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET
|
||||
?? process.env.TKMIND_DEEP_SEARCH_SECRET
|
||||
?? '',
|
||||
},
|
||||
available_tools: [
|
||||
'tkmind_search',
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
clampUserCapabilities,
|
||||
DEFAULT_USER_CAPABILITIES,
|
||||
normalizeCapabilityPatch,
|
||||
resolveMindSearchMcpEndpoint,
|
||||
resolveSandboxMcpNodeExecPath,
|
||||
resolveSandboxMcpServerPath,
|
||||
resolveSandboxMcpUserDataPgUrl,
|
||||
@@ -53,6 +54,17 @@ test('resolveSandboxMcpUserDataPgUrl preserves native URLs and honors an explici
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveMindSearchMcpEndpoint rewrites host loopback for goosed containers', () => {
|
||||
assert.equal(
|
||||
resolveMindSearchMcpEndpoint('http://127.0.0.1:20080/search'),
|
||||
'http://host.docker.internal:20080/search',
|
||||
);
|
||||
assert.equal(
|
||||
resolveMindSearchMcpEndpoint('https://search.internal/query'),
|
||||
'https://search.internal/query',
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveSandboxMcpNodeExecPath honors container-path override without host fs checks', () => {
|
||||
assert.equal(resolveSandboxMcpNodeExecPath('/usr/local/bin/node'), '/usr/local/bin/node');
|
||||
assert.equal(resolveSandboxMcpNodeExecPath(''), process.execPath);
|
||||
|
||||
+33
-1
@@ -6,6 +6,7 @@ import {
|
||||
createPortalGatewayResearchLlmResolver,
|
||||
} from './deep-search-engine.mjs';
|
||||
import { createDeepSearchStore } from './deep-search-store.mjs';
|
||||
import { searchGithubCode } from './mindsearch-providers.mjs';
|
||||
|
||||
function sendJson(res, status, body) {
|
||||
const payload = JSON.stringify(body);
|
||||
@@ -63,19 +64,50 @@ function canAccessTask(task, requestUserId) {
|
||||
export function createDeepSearchHttpServer({
|
||||
engine,
|
||||
secret = process.env.TKMIND_DEEP_SEARCH_SECRET || '',
|
||||
githubToken = process.env.GITHUB_TOKEN || '',
|
||||
githubSearch = searchGithubCode,
|
||||
logger = console,
|
||||
} = {}) {
|
||||
if (!engine) throw new Error('Deep Search engine is required');
|
||||
return http.createServer(async (req, res) => {
|
||||
const url = new URL(req.url ?? '/', 'http://deep-search.local');
|
||||
if (url.pathname === '/health' && req.method === 'GET') {
|
||||
return sendJson(res, 200, engine.getHealth());
|
||||
return sendJson(res, 200, {
|
||||
...engine.getHealth(),
|
||||
providers: {
|
||||
github: { configured: Boolean(githubToken) },
|
||||
},
|
||||
});
|
||||
}
|
||||
if (secret && req.headers['x-secret-key'] !== secret) {
|
||||
return sendJson(res, 401, { code: 'UNAUTHORIZED', message: 'Invalid service secret' });
|
||||
}
|
||||
const requestUserId = String(req.headers['x-user-id'] ?? '').trim();
|
||||
try {
|
||||
if (url.pathname === '/v1/providers/github/search' && req.method === 'POST') {
|
||||
if (!githubToken) {
|
||||
const error = new Error('GitHub search token is not configured');
|
||||
error.status = 503;
|
||||
throw error;
|
||||
}
|
||||
const input = await readJson(req, { maxBytes: 32_000 });
|
||||
const query = String(input.query ?? '').trim();
|
||||
if (!query || query.length > 500) {
|
||||
const error = new Error('query must be 1-500 characters');
|
||||
error.status = 400;
|
||||
throw error;
|
||||
}
|
||||
const limit = Math.max(1, Math.min(20, Number(input.limit ?? 10) || 10));
|
||||
const results = await githubSearch(query, {
|
||||
limit,
|
||||
token: githubToken,
|
||||
});
|
||||
return sendJson(res, 200, {
|
||||
query,
|
||||
results,
|
||||
service: 'tkmind-deep-search',
|
||||
});
|
||||
}
|
||||
if (url.pathname === '/v1/search' && req.method === 'POST') {
|
||||
const input = await readJson(req);
|
||||
const results = await engine.search(input);
|
||||
|
||||
+26
-1
@@ -281,7 +281,23 @@ test('Deep Search supports cancellation while a provider is running', async () =
|
||||
|
||||
test('Deep Search HTTP service exposes health, async start, status, search, and auth', async (t) => {
|
||||
const { engine, store } = createMockEngine({ id: 'task-http' });
|
||||
const server = createDeepSearchHttpServer({ engine, secret: 'local-secret', logger: { warn() {} } });
|
||||
let githubRequest;
|
||||
const server = createDeepSearchHttpServer({
|
||||
engine,
|
||||
secret: 'local-secret',
|
||||
githubToken: 'github-secret-token',
|
||||
githubSearch: async (query, options) => {
|
||||
githubRequest = { query, options };
|
||||
return [{
|
||||
title: 'tkmind/code:server.mjs',
|
||||
url: 'https://github.com/tkmind/code/blob/main/server.mjs',
|
||||
snippet: 'server',
|
||||
source: 'github',
|
||||
rank: 1,
|
||||
}];
|
||||
},
|
||||
logger: { warn() {} },
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
t.after(async () => {
|
||||
await new Promise((resolve) => server.close(resolve));
|
||||
@@ -291,8 +307,17 @@ test('Deep Search HTTP service exposes health, async start, status, search, and
|
||||
const base = `http://127.0.0.1:${address.port}`;
|
||||
const health = await fetch(`${base}/health`).then((response) => response.json());
|
||||
assert.equal(health.ok, true);
|
||||
assert.equal(health.providers.github.configured, true);
|
||||
assert.equal((await fetch(`${base}/v1/research`)).status, 401);
|
||||
const headers = { 'content-type': 'application/json', 'x-secret-key': 'local-secret' };
|
||||
const github = await fetch(`${base}/v1/providers/github/search`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ query: 'repo:tkmind/code server', limit: 3 }),
|
||||
}).then((response) => response.json());
|
||||
assert.equal(github.results.length, 1);
|
||||
assert.equal(githubRequest.options.token, 'github-secret-token');
|
||||
assert.equal(githubRequest.options.limit, 3);
|
||||
const search = await fetch(`${base}/v1/search`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
|
||||
@@ -30,6 +30,11 @@ npm run dev:deep-search
|
||||
|
||||
The server listens on `127.0.0.1:20100` by default.
|
||||
|
||||
On production host `103`, SearXNG is exposed at
|
||||
`http://127.0.0.1:20080/search`. Goosed containers reach host services through
|
||||
`host.docker.internal`; Portal rewrites loopback MindSearch service endpoints
|
||||
only when constructing the MCP extension environment.
|
||||
|
||||
## API
|
||||
|
||||
| Method | Path | Purpose |
|
||||
@@ -73,6 +78,7 @@ Start response:
|
||||
| `TKMIND_DEEP_SEARCH_SECRET` | empty | Optional `X-Secret-Key` required by non-health routes |
|
||||
| `TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL` | `http://127.0.0.1:8081/api/internal/deep-search/llm` | Portal LLM gateway |
|
||||
| `TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET` | `TKMIND_DEEP_SEARCH_SECRET` | Portal gateway bearer secret |
|
||||
| `GITHUB_TOKEN` | empty | GitHub code-search token, stored only in the standalone service environment |
|
||||
|
||||
## Isolation and safety
|
||||
|
||||
@@ -84,6 +90,40 @@ Start response:
|
||||
- Source reading rejects credentials, loopback, link-local, private, multicast, and private-DNS destinations.
|
||||
- Redirect targets are resolved and validated again.
|
||||
- Request bodies, result counts, source content, timeouts, and task depths are bounded.
|
||||
- GitHub Code Search is proxied by Deep Search so the provider token is never
|
||||
copied into goosed extension environments.
|
||||
|
||||
## Production runtime on 103
|
||||
|
||||
Deep Search is released independently from Portal and `memindadm`:
|
||||
|
||||
```bash
|
||||
npm run build:deep-search-runtime
|
||||
CONFIRMED_CI_SHA="$(git rev-parse HEAD)" \
|
||||
bash scripts/release-deep-search-runtime-prod.sh
|
||||
```
|
||||
|
||||
The production layout is:
|
||||
|
||||
```text
|
||||
/Users/john/Project/deep-search-runtime/
|
||||
├── current -> releases/<release-id>/
|
||||
├── releases/<release-id>/
|
||||
├── shared/.env
|
||||
├── data/research.sqlite
|
||||
├── logs/deep-search.log
|
||||
└── backups/<release-id>/
|
||||
```
|
||||
|
||||
The release workflow requires a clean `main` that exactly matches
|
||||
`origin/main`, a matching confirmed CI SHA, artifact checksum verification,
|
||||
SQLite online backup, LaunchAgent rollback, host health/search probes, and a
|
||||
Goose-container reachability probe. It never releases or restarts Portal or
|
||||
`memindadm`.
|
||||
|
||||
The installer creates a random service secret on first install. Add a newly
|
||||
issued, least-privilege GitHub token to `shared/.env` only after revoking any
|
||||
token that has appeared in chat or logs.
|
||||
|
||||
## MindSearch registration
|
||||
|
||||
|
||||
@@ -68,6 +68,27 @@ async function postServiceJson(endpoint, path, payload, {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function searchGithubGateway(query, {
|
||||
endpoint = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL,
|
||||
limit = 10,
|
||||
timeoutMs = 12000,
|
||||
fetchImpl = fetch,
|
||||
secret = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_SECRET
|
||||
|| process.env.TKMIND_DEEP_SEARCH_SECRET,
|
||||
} = {}) {
|
||||
const body = await postServiceJson(endpoint, '/v1/providers/github/search', {
|
||||
query,
|
||||
limit,
|
||||
}, {
|
||||
timeoutMs,
|
||||
fetchImpl,
|
||||
secret,
|
||||
});
|
||||
return (Array.isArray(body.results) ? body.results : [])
|
||||
.slice(0, limit)
|
||||
.map((item, index) => normalizeSearchResult({ ...item, source: 'github' }, index));
|
||||
}
|
||||
|
||||
export async function searchResearchService(query, {
|
||||
endpoint,
|
||||
type = 'web',
|
||||
|
||||
+59
-2
@@ -7,11 +7,15 @@ import {
|
||||
probeMindSearchService,
|
||||
resolveMindSearchService,
|
||||
} from './mindsearch-config.mjs';
|
||||
import { buildAgentExtensionPolicy, DEFAULT_USER_CAPABILITIES } from './capabilities.mjs';
|
||||
import {
|
||||
buildAgentExtensionPolicy,
|
||||
DEFAULT_USER_CAPABILITIES,
|
||||
} from './capabilities.mjs';
|
||||
import {
|
||||
cancelResearchTask,
|
||||
getResearchTask,
|
||||
searchGithubCode,
|
||||
searchGithubGateway,
|
||||
searchResearchService,
|
||||
searchSearxng,
|
||||
startResearchTask,
|
||||
@@ -102,12 +106,40 @@ test('MindSearch never changes legacy web extension and is gated by capability/c
|
||||
enabled: true,
|
||||
mode: 'assist',
|
||||
providers: {},
|
||||
services: [{ id: 'deep-search', enabled: true, adapter: 'research-http' }],
|
||||
settings: { searxngEndpoint: 'http://127.0.0.1:20080/search' },
|
||||
services: [
|
||||
{
|
||||
id: 'searxng',
|
||||
enabled: true,
|
||||
adapter: 'searxng',
|
||||
endpoint: 'http://127.0.0.1:20080/search',
|
||||
},
|
||||
{
|
||||
id: 'deep-search',
|
||||
enabled: true,
|
||||
adapter: 'research-http',
|
||||
endpoint: 'http://127.0.0.1:20100',
|
||||
},
|
||||
],
|
||||
routes: { research: 'deep-search' },
|
||||
},
|
||||
});
|
||||
const extension = policy.extensionOverrides.find((ext) => ext.name === 'tkmind-search');
|
||||
assert.equal(extension.envs.TKMIND_SEARCH_USER_ID, 'user-123');
|
||||
assert.equal(
|
||||
extension.envs.TKMIND_SEARCH_SEARXNG_URL,
|
||||
'http://host.docker.internal:20080/search',
|
||||
);
|
||||
assert.equal(
|
||||
extension.envs.TKMIND_SEARCH_GITHUB_GATEWAY_URL,
|
||||
'http://host.docker.internal:20100/',
|
||||
);
|
||||
assert.equal(extension.envs.GITHUB_TOKEN, undefined);
|
||||
const mcpServices = JSON.parse(extension.envs.TKMIND_SEARCH_SERVICES_JSON);
|
||||
assert.equal(
|
||||
mcpServices.find((service) => service.id === 'deep-search').endpoint,
|
||||
'http://host.docker.internal:20100/',
|
||||
);
|
||||
assert.ok(extension.available_tools.includes('tkmind_research_status'));
|
||||
assert.ok(extension.available_tools.includes('tkmind_research_cancel'));
|
||||
});
|
||||
@@ -125,6 +157,31 @@ test('GitHub code adapter sends bounded queries and normalizes results', async (
|
||||
assert.equal(result[0].source, 'github');
|
||||
});
|
||||
|
||||
test('GitHub gateway keeps the provider token outside the MCP process', async () => {
|
||||
let requested;
|
||||
const result = await searchGithubGateway('repo:openai goose', {
|
||||
endpoint: 'http://deep-search.local:20100',
|
||||
secret: 'gateway-secret',
|
||||
fetchImpl: async (url, options) => {
|
||||
requested = { url: String(url), options };
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
results: [{
|
||||
title: 'openai/goose:src/goose.rs',
|
||||
url: 'https://github.com/openai/goose/blob/main/src/goose.rs',
|
||||
snippet: 'agent',
|
||||
}],
|
||||
}),
|
||||
};
|
||||
},
|
||||
});
|
||||
assert.equal(requested.url, 'http://deep-search.local:20100/v1/providers/github/search');
|
||||
assert.equal(requested.options.headers['x-secret-key'], 'gateway-secret');
|
||||
assert.equal(requested.options.headers.authorization, undefined);
|
||||
assert.equal(result[0].source, 'github');
|
||||
});
|
||||
|
||||
test('Research service adapter uses the stable HTTP contract', async () => {
|
||||
const requests = [];
|
||||
const fetchImpl = async (url, options) => {
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
"backfill:plaza-public-urls": "node scripts/backfill-plaza-public-urls.mjs",
|
||||
"enrich:plaza": "node scripts/enrich-plaza-posts.mjs",
|
||||
"build": "vite build",
|
||||
"build:deep-search-runtime": "node scripts/build-deep-search-runtime.mjs",
|
||||
"build:portal-runtime": "node scripts/build-portal-runtime.mjs",
|
||||
"build:imgproxy-runtime": "bash scripts/build-imgproxy-runtime-image.sh",
|
||||
"build:mindspace-service-runtime": "node scripts/build-mindspace-service-runtime.mjs",
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import {
|
||||
chmod,
|
||||
copyFile,
|
||||
mkdir,
|
||||
rm,
|
||||
writeFile,
|
||||
} from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const runtimeRoot = path.join(root, '.runtime', 'deep-search');
|
||||
const esbuildBin = path.join(root, 'node_modules', '.bin', 'esbuild');
|
||||
const runtimeNodeTarget = process.env.DEEP_SEARCH_RUNTIME_NODE_TARGET || 'node24';
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: root,
|
||||
stdio: 'inherit',
|
||||
env: process.env,
|
||||
});
|
||||
child.once('error', reject);
|
||||
child.once('exit', (code, signal) => {
|
||||
if (code === 0) return resolve();
|
||||
reject(new Error(`${command} exited with ${code ?? signal}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
await rm(runtimeRoot, { recursive: true, force: true });
|
||||
await mkdir(path.join(runtimeRoot, 'scripts'), { recursive: true });
|
||||
|
||||
await run(esbuildBin, [
|
||||
'deep-search-server.mjs',
|
||||
'--bundle',
|
||||
'--platform=node',
|
||||
'--format=esm',
|
||||
`--target=${runtimeNodeTarget}`,
|
||||
'--outfile=.runtime/deep-search/deep-search-server.mjs',
|
||||
'--banner:js=import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);',
|
||||
]);
|
||||
|
||||
for (const script of [
|
||||
'run-deep-search-prod.sh',
|
||||
'install-deep-search-runtime-prod.sh',
|
||||
]) {
|
||||
const source = path.join(root, 'scripts', script);
|
||||
const target = path.join(runtimeRoot, 'scripts', script);
|
||||
await copyFile(source, target);
|
||||
await chmod(target, 0o755);
|
||||
}
|
||||
|
||||
await writeFile(
|
||||
path.join(runtimeRoot, 'RUNBOOK.txt'),
|
||||
[
|
||||
'TKMind Deep Search standalone runtime',
|
||||
'',
|
||||
'Production base: /Users/john/Project/deep-search-runtime',
|
||||
'Persistent env: shared/.env',
|
||||
'Persistent data: data/research.sqlite',
|
||||
'Release code: releases/<release-id>/',
|
||||
'Active release: current -> releases/<release-id>/',
|
||||
'LaunchAgent: cn.tkmind.memind-deep-search',
|
||||
'Health: http://127.0.0.1:20100/health',
|
||||
'SearXNG: http://127.0.0.1:20080/search',
|
||||
'',
|
||||
'The GitHub token stays in shared/.env and is never passed to goosed MCP processes.',
|
||||
'Portal and memindadm releases are intentionally separate.',
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
|
||||
console.log(`Deep Search runtime built at ${runtimeRoot}`);
|
||||
@@ -0,0 +1,31 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
const read = (relativePath) => readFileSync(path.join(root, relativePath), 'utf8');
|
||||
|
||||
test('Deep Search production runtime is isolated from Portal releases', () => {
|
||||
const builder = read('scripts/build-deep-search-runtime.mjs');
|
||||
assert.match(builder, /deep-search-server\.mjs/);
|
||||
assert.match(builder, /\.runtime', 'deep-search/);
|
||||
|
||||
const runner = read('scripts/run-deep-search-prod.sh');
|
||||
assert.match(runner, /shared\/\.env/);
|
||||
assert.match(runner, /data\/research\.sqlite/);
|
||||
assert.match(runner, /127\.0\.0\.1:20080\/search/);
|
||||
|
||||
const installer = read('scripts/install-deep-search-runtime-prod.sh');
|
||||
assert.match(installer, /cn\.tkmind\.memind-deep-search/);
|
||||
assert.match(installer, /chmod 600/);
|
||||
assert.match(installer, /openssl rand -hex 32/);
|
||||
|
||||
const release = read('scripts/release-deep-search-runtime-prod.sh');
|
||||
assert.match(release, /branch.*main/);
|
||||
assert.match(release, /CONFIRMED_CI_SHA/);
|
||||
assert.match(release, /shasum -a 256/);
|
||||
assert.match(release, /rollback/);
|
||||
assert.doesNotMatch(release, /release-portal-runtime-prod/);
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runs on 103 from a verified Deep Search release directory.
|
||||
set -euo pipefail
|
||||
|
||||
RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}"
|
||||
RELEASE_DIR="${DEEP_SEARCH_RELEASE_DIR:?DEEP_SEARCH_RELEASE_DIR is required}"
|
||||
ENV_DIR="${RUNTIME_BASE}/shared"
|
||||
ENV_FILE="${ENV_DIR}/.env"
|
||||
DATA_DIR="${RUNTIME_BASE}/data"
|
||||
LOG_DIR="${RUNTIME_BASE}/logs"
|
||||
PLIST="${HOME}/Library/LaunchAgents/cn.tkmind.memind-deep-search.plist"
|
||||
LABEL="cn.tkmind.memind-deep-search"
|
||||
GUI="gui/$(id -u)"
|
||||
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||
|
||||
[[ -x "${NODE_BIN}" ]] || {
|
||||
echo "missing Node.js runtime: ${NODE_BIN}" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -f "${RELEASE_DIR}/deep-search-server.mjs" ]] || {
|
||||
echo "missing Deep Search server in ${RELEASE_DIR}" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
mkdir -p "${ENV_DIR}" "${DATA_DIR}" "${LOG_DIR}" "$(dirname "${PLIST}")"
|
||||
touch "${ENV_FILE}"
|
||||
chmod 600 "${ENV_FILE}"
|
||||
|
||||
ensure_env() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
if ! grep -qE "^${key}=" "${ENV_FILE}"; then
|
||||
printf '%s=%s\n' "${key}" "${value}" >> "${ENV_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
if ! grep -qE '^TKMIND_DEEP_SEARCH_SECRET=.+$' "${ENV_FILE}"; then
|
||||
secret="$(openssl rand -hex 32)"
|
||||
ensure_env TKMIND_DEEP_SEARCH_SECRET "${secret}"
|
||||
fi
|
||||
ensure_env TKMIND_DEEP_SEARCH_HOST "127.0.0.1"
|
||||
ensure_env TKMIND_DEEP_SEARCH_PORT "20100"
|
||||
ensure_env TKMIND_DEEP_SEARCH_DB "${DATA_DIR}/research.sqlite"
|
||||
ensure_env TKMIND_DEEP_SEARCH_SEARXNG_URL "http://127.0.0.1:20080/search"
|
||||
ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL "http://127.0.0.1:8081/api/internal/deep-search/llm"
|
||||
ensure_env TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET '${TKMIND_DEEP_SEARCH_SECRET}'
|
||||
|
||||
ln -sfn "${RELEASE_DIR}" "${RUNTIME_BASE}/current"
|
||||
|
||||
cat > "${PLIST}" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0"><dict>
|
||||
<key>Label</key><string>${LABEL}</string>
|
||||
<key>ProgramArguments</key><array><string>${RUNTIME_BASE}/current/scripts/run-deep-search-prod.sh</string></array>
|
||||
<key>WorkingDirectory</key><string>${RUNTIME_BASE}/current</string>
|
||||
<key>EnvironmentVariables</key><dict>
|
||||
<key>DEEP_SEARCH_RUNTIME_BASE</key><string>${RUNTIME_BASE}</string>
|
||||
<key>NODE_BIN</key><string>${NODE_BIN}</string>
|
||||
</dict>
|
||||
<key>RunAtLoad</key><true/>
|
||||
<key>KeepAlive</key><true/>
|
||||
<key>ProcessType</key><string>Background</string>
|
||||
<key>ThrottleInterval</key><integer>10</integer>
|
||||
<key>StandardOutPath</key><string>${LOG_DIR}/deep-search.log</string>
|
||||
<key>StandardErrorPath</key><string>${LOG_DIR}/deep-search.log</string>
|
||||
</dict></plist>
|
||||
EOF
|
||||
|
||||
plutil -lint "${PLIST}" >/dev/null
|
||||
launchctl bootout "${GUI}/${LABEL}" >/dev/null 2>&1 || true
|
||||
launchctl bootstrap "${GUI}" "${PLIST}"
|
||||
launchctl enable "${GUI}/${LABEL}"
|
||||
launchctl kickstart -k "${GUI}/${LABEL}"
|
||||
|
||||
for _ in $(seq 1 30); do
|
||||
if curl -fsS --max-time 2 "http://127.0.0.1:20100/health" >/dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Deep Search health check timed out" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bash
|
||||
# Publishes only the independently versioned Deep Search runtime to 103.
|
||||
# Portal and memindadm are not modified by this workflow.
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
HOST="${STUDIO_HOST:-58.38.22.103}"
|
||||
REMOTE_ROOT="${STUDIO_REMOTE_ROOT:-/Users/john/Project}"
|
||||
RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-${REMOTE_ROOT}/deep-search-runtime}"
|
||||
INCOMING="${REMOTE_ROOT}/incoming/deep-search-runtime"
|
||||
RELEASE_ID="$(date +%Y%m%d-%H%M%S)-$(git -C "${ROOT}" rev-parse --short HEAD)"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/deep-search-runtime-release.XXXXXX")"
|
||||
DRY_RUN=0
|
||||
AUTO_YES=0
|
||||
|
||||
cleanup() {
|
||||
rm -rf "${TMP_DIR}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
--yes) AUTO_YES=1 ;;
|
||||
-h|--help)
|
||||
cat <<'EOF'
|
||||
Usage: bash scripts/release-deep-search-runtime-prod.sh [--dry-run] [--yes]
|
||||
|
||||
Required production gates:
|
||||
- current branch is main
|
||||
- HEAD exactly matches origin/main
|
||||
- worktree is clean
|
||||
- CONFIRMED_CI_SHA equals the full current HEAD
|
||||
|
||||
This installs only Deep Search on 103. It does not release Portal or memindadm.
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown argument: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
git -C "${ROOT}" fetch origin main --quiet
|
||||
branch="$(git -C "${ROOT}" branch --show-current)"
|
||||
head_sha="$(git -C "${ROOT}" rev-parse HEAD)"
|
||||
origin_sha="$(git -C "${ROOT}" rev-parse origin/main)"
|
||||
[[ "${branch}" == "main" ]] || {
|
||||
echo "Deep Search production release must run from main" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "${head_sha}" == "${origin_sha}" ]] || {
|
||||
echo "main must exactly match origin/main" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ -z "$(git -C "${ROOT}" status --porcelain=v1 --untracked-files=all)" ]] || {
|
||||
echo "worktree must be clean" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "${CONFIRMED_CI_SHA:-}" == "${head_sha}" ]] || {
|
||||
echo "CONFIRMED_CI_SHA must equal the verified main HEAD: ${head_sha}" >&2
|
||||
exit 1
|
||||
}
|
||||
ALLOW_MAIN_RELEASE=1 bash "${ROOT}/scripts/check-release-ready.sh"
|
||||
|
||||
(
|
||||
cd "${ROOT}"
|
||||
npm run test:deep-search
|
||||
node --test capabilities.test.mjs scripts/deep-search-runtime-package.test.mjs
|
||||
npm run build:deep-search-runtime
|
||||
)
|
||||
|
||||
RUNTIME_DIR="${ROOT}/.runtime/deep-search"
|
||||
for file in \
|
||||
deep-search-server.mjs \
|
||||
scripts/run-deep-search-prod.sh \
|
||||
scripts/install-deep-search-runtime-prod.sh \
|
||||
RUNBOOK.txt; do
|
||||
[[ -f "${RUNTIME_DIR}/${file}" ]] || {
|
||||
echo "missing runtime artifact: ${file}" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
MANIFEST="${RUNTIME_DIR}/release-manifest.txt"
|
||||
{
|
||||
echo "release_id=${RELEASE_ID}"
|
||||
echo "git_head=${head_sha}"
|
||||
echo "git_branch=${branch}"
|
||||
echo "created_at=$(date '+%Y-%m-%d %H:%M:%S %z')"
|
||||
echo "artifact=.runtime/deep-search"
|
||||
echo "persistent_env=${RUNTIME_BASE}/shared/.env"
|
||||
echo "persistent_db=${RUNTIME_BASE}/data/research.sqlite"
|
||||
} > "${MANIFEST}"
|
||||
|
||||
ARCHIVE="${TMP_DIR}/deep-search-runtime-${RELEASE_ID}.tar.gz"
|
||||
SHA_FILE="${ARCHIVE}.sha256"
|
||||
tar -C "${RUNTIME_DIR}" -czf "${ARCHIVE}" .
|
||||
(cd "${TMP_DIR}" && shasum -a 256 "$(basename "${ARCHIVE}")" > "$(basename "${SHA_FILE}")")
|
||||
|
||||
if [[ "${DRY_RUN}" -eq 1 ]]; then
|
||||
shasum -a 256 -c "${SHA_FILE}"
|
||||
echo "Deep Search dry-run artifact verified: ${ARCHIVE}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ "${AUTO_YES}" -ne 1 ]]; then
|
||||
echo "Target: ${HOST}:${RUNTIME_BASE}"
|
||||
echo "Release: ${RELEASE_ID}"
|
||||
read -r -p "Install standalone Deep Search on 103? [y/N] " confirm </dev/tty
|
||||
[[ "${confirm}" =~ ^[Yy]$ ]] || exit 0
|
||||
fi
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=15 "${HOST}" "mkdir -p '${INCOMING}' '${RUNTIME_BASE}/releases' '${RUNTIME_BASE}/backups'"
|
||||
scp -q "${ARCHIVE}" "${SHA_FILE}" "${HOST}:${INCOMING}/"
|
||||
|
||||
ssh -o BatchMode=yes "${HOST}" \
|
||||
"RELEASE_ID='${RELEASE_ID}' RUNTIME_BASE='${RUNTIME_BASE}' INCOMING='${INCOMING}' /bin/bash" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
|
||||
archive="${INCOMING}/deep-search-runtime-${RELEASE_ID}.tar.gz"
|
||||
sha_file="${archive}.sha256"
|
||||
release_dir="${RUNTIME_BASE}/releases/${RELEASE_ID}"
|
||||
backup_dir="${RUNTIME_BASE}/backups/${RELEASE_ID}"
|
||||
current_link="${RUNTIME_BASE}/current"
|
||||
plist="${HOME}/Library/LaunchAgents/cn.tkmind.memind-deep-search.plist"
|
||||
label="cn.tkmind.memind-deep-search"
|
||||
gui="gui/$(id -u)"
|
||||
previous_target="$(readlink "${current_link}" 2>/dev/null || true)"
|
||||
database_path="${RUNTIME_BASE}/data/research.sqlite"
|
||||
|
||||
mkdir -p "${release_dir}" "${backup_dir}"
|
||||
cp "${plist}" "${backup_dir}/launchagent.plist" 2>/dev/null || true
|
||||
if [[ -f "${database_path}" ]]; then
|
||||
sqlite3 "${database_path}" ".timeout 10000" \
|
||||
".backup '${backup_dir}/research.sqlite'" >/dev/null
|
||||
fi
|
||||
cd "${INCOMING}"
|
||||
shasum -a 256 -c "$(basename "${sha_file}")"
|
||||
tar -xzf "${archive}" -C "${release_dir}"
|
||||
|
||||
rollback() {
|
||||
if [[ -n "${previous_target}" && -d "${previous_target}" ]]; then
|
||||
ln -sfn "${previous_target}" "${current_link}"
|
||||
else
|
||||
rm -f "${current_link}"
|
||||
fi
|
||||
if [[ -f "${backup_dir}/launchagent.plist" ]]; then
|
||||
cp "${backup_dir}/launchagent.plist" "${plist}"
|
||||
launchctl bootout "${gui}/${label}" >/dev/null 2>&1 || true
|
||||
launchctl bootstrap "${gui}" "${plist}" >/dev/null 2>&1 || true
|
||||
launchctl kickstart -k "${gui}/${label}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
trap rollback ERR
|
||||
|
||||
DEEP_SEARCH_RUNTIME_BASE="${RUNTIME_BASE}" \
|
||||
DEEP_SEARCH_RELEASE_DIR="${release_dir}" \
|
||||
bash "${release_dir}/scripts/install-deep-search-runtime-prod.sh"
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${RUNTIME_BASE}/shared/.env"
|
||||
set +a
|
||||
|
||||
curl -fsS --max-time 5 "http://127.0.0.1:20100/health" >/dev/null
|
||||
curl -fsS --max-time 20 \
|
||||
-H "X-Secret-Key: ${TKMIND_DEEP_SEARCH_SECRET}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"query":"OpenAI","limit":3}' \
|
||||
"http://127.0.0.1:20100/v1/search" \
|
||||
| grep -q '"results"'
|
||||
|
||||
docker_bin=""
|
||||
for candidate in /usr/local/bin/docker /opt/homebrew/bin/docker; do
|
||||
if [[ -x "${candidate}" ]]; then
|
||||
docker_bin="${candidate}"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ -n "${docker_bin}" ]]; then
|
||||
goosed_container="$("${docker_bin}" ps --format '{{.Names}}' | grep -E '^goosed-prod-' | head -1)"
|
||||
if [[ -n "${goosed_container}" ]]; then
|
||||
"${docker_bin}" exec "${goosed_container}" sh -lc \
|
||||
'curl -fsS --max-time 5 http://host.docker.internal:20100/health >/dev/null'
|
||||
fi
|
||||
fi
|
||||
|
||||
trap - ERR
|
||||
printf '%s\n' "${RELEASE_ID}" > "${RUNTIME_BASE}/current-release.txt"
|
||||
REMOTE
|
||||
|
||||
echo "Deep Search installed and verified on 103: ${RELEASE_ID}"
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
RUNTIME_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
RUNTIME_BASE="${DEEP_SEARCH_RUNTIME_BASE:-/Users/john/Project/deep-search-runtime}"
|
||||
ENV_FILE="${DEEP_SEARCH_ENV_FILE:-${RUNTIME_BASE}/shared/.env}"
|
||||
|
||||
if [[ ! -f "${ENV_FILE}" ]]; then
|
||||
echo "[deep-search-start] missing environment file: ${ENV_FILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "${ENV_FILE}"
|
||||
set +a
|
||||
|
||||
: "${TKMIND_DEEP_SEARCH_SECRET:?missing TKMIND_DEEP_SEARCH_SECRET}"
|
||||
|
||||
export NODE_ENV=production
|
||||
export TKMIND_DEEP_SEARCH_HOST="${TKMIND_DEEP_SEARCH_HOST:-127.0.0.1}"
|
||||
export TKMIND_DEEP_SEARCH_PORT="${TKMIND_DEEP_SEARCH_PORT:-20100}"
|
||||
export TKMIND_DEEP_SEARCH_DB="${TKMIND_DEEP_SEARCH_DB:-${RUNTIME_BASE}/data/research.sqlite}"
|
||||
export TKMIND_DEEP_SEARCH_SEARXNG_URL="${TKMIND_DEEP_SEARCH_SEARXNG_URL:-http://127.0.0.1:20080/search}"
|
||||
export TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_URL:-http://127.0.0.1:8081/api/internal/deep-search/llm}"
|
||||
export TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET="${TKMIND_DEEP_SEARCH_LLM_GATEWAY_SECRET:-${TKMIND_DEEP_SEARCH_SECRET}}"
|
||||
|
||||
mkdir -p "$(dirname "${TKMIND_DEEP_SEARCH_DB}")" "${RUNTIME_BASE}/logs"
|
||||
|
||||
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||
if [[ ! -x "${NODE_BIN}" ]]; then
|
||||
NODE_BIN="$(command -v node)"
|
||||
fi
|
||||
|
||||
exec "${NODE_BIN}" "${RUNTIME_ROOT}/deep-search-server.mjs"
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
getResearchTask,
|
||||
readSafeUrl,
|
||||
searchGithubCode,
|
||||
searchGithubGateway,
|
||||
searchResearchService,
|
||||
searchSearxng,
|
||||
startResearchTask,
|
||||
@@ -86,7 +87,13 @@ function handle(message) {
|
||||
const route = checked.value.type === 'code' ? 'code' : checked.value.type === 'news' ? 'news' : 'web';
|
||||
const service = resolveMindSearchService(config, route);
|
||||
if (service?.adapter === 'github') {
|
||||
return searchGithubCode(checked.value.query, { limit: checked.value.limit, endpoint: service.endpoint || undefined }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message));
|
||||
const operation = process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL
|
||||
? searchGithubGateway
|
||||
: searchGithubCode;
|
||||
return operation(checked.value.query, {
|
||||
limit: checked.value.limit,
|
||||
endpoint: process.env.TKMIND_SEARCH_GITHUB_GATEWAY_URL || service.endpoint || undefined,
|
||||
}).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message));
|
||||
}
|
||||
if (service?.adapter === 'searxng') {
|
||||
return searchSearxng(checked.value.query, { limit: checked.value.limit, endpoint: service.endpoint }).then((results) => response(id, { content: [{ type: 'text', text: JSON.stringify({ results, query: checked.value.query, provider: service.id }) }], structuredContent: { results, query: checked.value.query, provider: service.id } })).catch((err) => error(id, SEARCH_ERROR_CODES.PROVIDER_UNAVAILABLE, err.message));
|
||||
|
||||
Reference in New Issue
Block a user