tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: harden orchestrator execution runtime

Browse Source
This commit is contained in:
john
2026-07-24 23:53:32 +08:00
parent 396bb78200
commit f6f2cd0933
55 changed files with 5122 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server.mjs
+133 -18
View File
@@ -31,6 +31,10 @@ import {
import { createTkmindProxy, sanitizeSessionConversationPublicHtmlLinks } from './tkmind-proxy.mjs';
import { createSessionAccess, isSessionBrokerEnabled } from './session-broker.mjs';
import { isSessionBrokerMetricsEnabled } from './session-broker-metrics.mjs';
import {
cancelSessionActiveRequest,
quiesceSessionStdioExtensions,
} from './session-runtime-lifecycle.mjs';
import {
clearUserLogoutCookies,
clearUserSessionCookie,
@@ -45,6 +49,7 @@ import { isLocalDevHostname } from './scripts/local-test-config.mjs';
import { PUBLISH_ROOT_DIR, PUBLISH_KEY_UUID, PUBLIC_ZONE_DIR } from './user-publish.mjs';
import { ensureWorkspaceHtmlThumbnail, startWorkspaceThumbnailWatcher, workspaceThumbnailRelativePath } from './mindspace-workspace-thumbnails.mjs';
import { injectMindSpaceAnalytics, resolveAnalyticsOwnerLabel, resolveAnalyticsOwnerSegment, resolveMindSpaceAnalyticsConfig, sendMindSpaceAnalyticsEvent } from './mindspace-analytics.mjs';
import { injectMindSpaceRybbit, resolveMindSpaceRybbitConfig, sendMindSpaceRybbitEvent } from './mindspace-rybbit.mjs';
import { startWorkspaceAssetSyncWatcher } from './mindspace-workspace-sync.mjs';
import { attachRequestId, sendData, sendError } from './api-response.mjs';
import { createNotificationDispatcher } from './notification-dispatcher.mjs';
@@ -71,6 +76,7 @@ import {
buildPublishedHtmlViewContext,
injectPublishedPageDataContext,
parseMindSpacePublishFilePath,
resolveMindSpacePageDataContext,
resolvePublicRequestOrigin,
} from './mindspace-public-page-context.mjs';
import {
@@ -256,6 +262,17 @@ const HOST = String(process.env.H5_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
const API_TARGETS = parseApiTargets();
const API_TARGET = API_TARGETS[0] ?? 'https://127.0.0.1:18006';
const API_SECRET = process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret';
const AGENT_RUN_WORKER_RUNTIME_ROOT = fs.realpathSync(__dirname);
const AGENT_RUN_WORKER_BUILD_ID =
String(
process.env.MEMIND_RUNTIME_BUILD_ID
?? process.env.MEMIND_RELEASE_ID
?? process.env.GIT_COMMIT
?? 'dev',
).trim() || 'dev';
const AGENT_RUN_WORKER_ID =
String(process.env.MEMIND_AGENT_RUN_WORKER_ID ?? '').trim()
|| `${process.pid}:${crypto.randomUUID()}`;
const INTERNAL_AGENT_SECRET = process.env.MINDSPACE_INTERNAL_AGENT_SECRET ?? API_SECRET;
const DEEP_SEARCH_INTERNAL_SECRET = process.env.TKMIND_DEEP_SEARCH_SECRET ?? INTERNAL_AGENT_SECRET;
const mindSpaceServerRuntime = resolveMindSpaceServerRuntimeOptions(__dirname, process.env);
@@ -428,6 +445,54 @@ let authPool = null;
let pageDataService = null;
let pageDataPublicService = null;
let mindSpaceAnalyticsConfig = resolveMindSpaceAnalyticsConfig();
let mindSpaceRybbitConfig = resolveMindSpaceRybbitConfig();
let agentRunRecoveryTimer = null;
function startAgentRunRecoveryLoop() {
if (!agentRunGateway?.dispatchQueuedRuns || agentRunRecoveryTimer) return;
const configuredInterval = Number(process.env.MEMIND_AGENT_RUN_RECOVERY_INTERVAL_MS ?? 30_000);
const intervalMs = Number.isFinite(configuredInterval)
? Math.max(5_000, configuredInterval)
: 30_000;
const configuredHeartbeatStaleMs = Number(
process.env.MEMIND_AGENT_RUN_HEARTBEAT_STALE_MS ?? 90_000,
);
const heartbeatStaleMs = Number.isFinite(configuredHeartbeatStaleMs)
? Math.max(intervalMs * 2, configuredHeartbeatStaleMs)
: Math.max(intervalMs * 2, 90_000);
let sweepActive = false;
const sweep = async () => {
if (sweepActive) return;
sweepActive = true;
try {
const heartbeatRecovery = await agentRunGateway.recoverStaleRunningRuns({
staleMs: heartbeatStaleMs,
limit: 20,
dryRun: false,
reason: 'worker_heartbeat_stale_recovery',
});
const result = await agentRunGateway.dispatchQueuedRuns();
const recovered =
Number(heartbeatRecovery?.recovered ?? 0)
+ Number(result?.staleRecovery?.recovered ?? 0);
if (recovered > 0) {
console.warn(`[AgentRun] recovered ${recovered} stale run(s)`);
}
} catch (err) {
console.warn(
'[AgentRun] background recovery sweep failed:',
err instanceof Error ? err.message : err,
);
} finally {
sweepActive = false;
}
};
void sweep();
agentRunRecoveryTimer = setInterval(sweep, intervalMs);
agentRunRecoveryTimer.unref?.();
}
async function bootstrapUserAuth() {
try {
@@ -866,12 +931,34 @@ async function bootstrapUserAuth() {
}))],
};
},
quiesceSessionOnTerminal: async ({ sessionId, requestId, status }) => {
if (isDirectChatSessionId(sessionId)) {
return { removed: [], skipped: true };
}
const target = await tkmindProxy.resolveTarget(sessionId);
const sessionApiFetch = (pathname, init) =>
tkmindProxy.apiFetchTo(target, pathname, init);
const cancellation = status === 'failed'
? await cancelSessionActiveRequest(sessionApiFetch, sessionId, requestId)
: { cancelled: false, skipped: true };
const quiesced = await quiesceSessionStdioExtensions(
sessionApiFetch,
sessionId,
);
return { ...quiesced, cancellation };
},
autoDispatch: ['1', 'true', 'yes', 'on'].includes(
String(process.env.MEMIND_AGENT_RUN_AUTODISPATCH ?? '1').trim().toLowerCase(),
),
maxConcurrentRuns: Number(process.env.MEMIND_AGENT_RUN_QUEUE_CONCURRENCY ?? 1),
runTimeoutMs: Number(process.env.MEMIND_AGENT_RUN_TIMEOUT_MS ?? 15 * 60 * 1000),
workerIdentity: {
workerId: AGENT_RUN_WORKER_ID,
runtimeRoot: AGENT_RUN_WORKER_RUNTIME_ROOT,
buildId: AGENT_RUN_WORKER_BUILD_ID,
},
});
startAgentRunRecoveryLoop();
const wechatMp = await loadWechatMpModule(__dirname);
wechatMpService = wechatMp.createWechatMpService({
config: WECHAT_MP_CONFIG,
@@ -901,17 +988,25 @@ async function bootstrapUserAuth() {
sessionIntentClassifier: ({ text }) => chatIntentRouter?.classifySessionAction({ text }),
onPageGenerated: async ({ userId, sessionId, artifacts = [] }) => {
for (const artifact of artifacts) {
void sendMindSpaceAnalyticsEvent({
config: mindSpaceAnalyticsConfig,
const pageOwner = await userAuth?.getUserById(userId).catch(() => null) ?? {};
const analyticsPayload = {
eventName: 'page_generated',
ownerId: userId,
ownerSegment: resolveAnalyticsOwnerSegment(await userAuth?.getUserById(userId).catch(() => null) ?? {}),
ownerLabel: resolveAnalyticsOwnerLabel(await userAuth?.getUserById(userId).catch(() => null) ?? {}),
ownerSegment: resolveAnalyticsOwnerSegment(pageOwner),
ownerLabel: resolveAnalyticsOwnerLabel(pageOwner),
pageId: artifact.relativePath,
publicationId: sessionId,
agentRunId: sessionId,
channel: 'wechat_mp',
url: artifact.url || artifact.relativePath || '/',
};
void sendMindSpaceAnalyticsEvent({
config: mindSpaceAnalyticsConfig,
...analyticsPayload,
});
void sendMindSpaceRybbitEvent({
config: mindSpaceRybbitConfig,
...analyticsPayload,
});
}
},
@@ -5356,8 +5451,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
const absolutePath = path.resolve(publishDir, relativePath);
if (generationAnalyticsEvents.has(relativePath) || !fs.existsSync(absolutePath)) continue;
generationAnalyticsEvents.add(relativePath);
void sendMindSpaceAnalyticsEvent({
config: mindSpaceAnalyticsConfig,
const analyticsPayload = {
eventName: 'page_generated',
ownerId: req.currentUser.id,
ownerSegment: resolveAnalyticsOwnerSegment(req.currentUser),
@@ -5367,6 +5461,14 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
agentRunId: sid,
channel: 'h5',
url: `/${PUBLISH_ROOT_DIR}/${req.currentUser.id}/${relativePath}`,
};
void sendMindSpaceAnalyticsEvent({
config: mindSpaceAnalyticsConfig,
...analyticsPayload,
});
void sendMindSpaceRybbitEvent({
config: mindSpaceRybbitConfig,
...analyticsPayload,
});
}
};
@@ -5587,6 +5689,15 @@ app.use('/analytics', createProxyMiddleware({
secure: false,
pathRewrite: { [`^${mindSpaceAnalyticsConfig.hostPath}`]: '' },
}));
app.use('/rybbit', createProxyMiddleware({
target: 'https://rybbit.tkmind.cn',
router: () => mindSpaceRybbitConfig.rybbitUrl || process.env.MEMIND_RYBBIT_URL || process.env.RYBBIT_URL || 'https://rybbit.tkmind.cn',
changeOrigin: true,
secure: true,
// Express strips the /rybbit mount, so /rybbit/script.js arrives as /script.js.
// Rybbit serves trackers under /api/*, so re-prefix here.
pathRewrite: { '^/': '/api/' },
}));
// Express routing is case-insensitive by default, so the lowercase /mindspace API
// mount would otherwise capture public /MindSpace/... page URLs.
app.use(`/${PUBLISH_ROOT_DIR}`, async (req, res, next) => {
@@ -6427,18 +6538,14 @@ async function sendPublishFile(req, res, filePath, { isOwner = true } = {}) {
? await userAuth.getUserById(parsedPublishPath.userId).catch(() => null)
: null;
let pageDataContext = null;
if (mindSpacePages) {
if (parsedPublishPath?.userId && parsedPublishPath.relativePath) {
const page = await mindSpacePages
.findPageByRelativePath(parsedPublishPath.userId, parsedPublishPath.relativePath)
.catch(() => null);
if (page?.id) {
pageDataContext = {
pageId: page.id,
accessMode: page.publicationAccessMode ?? null,
};
}
}
if (parsedPublishPath?.userId && parsedPublishPath.relativePath) {
pageDataContext = await resolveMindSpacePageDataContext({
pool: authPool,
pageService: mindSpacePages,
userId: parsedPublishPath.userId,
relativePath: parsedPublishPath.relativePath,
logger: console,
});
}
html = injectMindSpaceAnalytics(html, {
ownerId: parsedPublishPath?.userId ?? '',
@@ -6448,6 +6555,14 @@ async function sendPublishFile(req, res, filePath, { isOwner = true } = {}) {
publicationId: pageDataContext?.publicationId ?? pageDataContext?.publication_id ?? '',
config: mindSpaceAnalyticsConfig,
});
html = injectMindSpaceRybbit(html, {
ownerId: parsedPublishPath?.userId ?? '',
ownerSegment: resolveAnalyticsOwnerSegment(pageOwner ?? {}),
ownerLabel: resolveAnalyticsOwnerLabel(pageOwner ?? {}),
pageId: pageDataContext?.pageId ?? '',
publicationId: pageDataContext?.publicationId ?? pageDataContext?.publication_id ?? '',
config: mindSpaceRybbitConfig,
});
const decorated = decorateMindSpacePublishedHtml({
html,
embed,