chore: add runtime observability ops
This commit is contained in:
John
@@ -28,10 +28,13 @@ Key runtime differences must stay in .env, not in the artifact:
|
||||
H5_USERS_ROOT / MINDSPACE_STORAGE_ROOT / MEMIND_SHARED_PUBLISH_ROOT
|
||||
|
||||
Deployment and operations transport:
|
||||
103 / Studio fixed IP: 58.38.22.103
|
||||
H5 public domain: mm.tkmind.cn
|
||||
2026-07-02 routing decision:
|
||||
- Temporarily move H5 public base from m.tkmind.cn to mm.tkmind.cn.
|
||||
- Future H5 public traffic must not depend on the 105 nginx -> 127.0.0.1:19081 -> reverse SSH tunnel -> Portal :8081 path.
|
||||
- Keep legacy 105/tunnel scripts only for rollback or explicitly requested migration work.
|
||||
Do not switch back to 10.10.* LAN paths unless explicitly required.
|
||||
Current public path: mm.tkmind.cn -> local nginx -> Portal :8081
|
||||
Legacy rollback-only path: m.tkmind.cn -> 105 nginx -> reverse SSH tunnel -> Portal :8081
|
||||
Future H5 traffic must not depend on 105 forwarding unless explicitly rolling back.
|
||||
|
||||
Streaming runtime operations:
|
||||
node scripts/check-stream-runtime.mjs
|
||||
node scripts/runtime-worker-drain.mjs status
|
||||
node scripts/runtime-worker-drain.mjs drain goosed-3
|
||||
node scripts/runtime-worker-drain.mjs undrain goosed-3
|
||||
|
||||
@@ -1,5 +1,21 @@
|
||||
{
|
||||
"name": "tkmind-h5-portal-runtime",
|
||||
"private": true,
|
||||
"type": "module"
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"dependencies": {
|
||||
"@node-rs/argon2": "^2.0.2",
|
||||
"@resvg/resvg-js": "^2.6.2",
|
||||
"debug": "^4.4.3",
|
||||
"express": "^4.21.2",
|
||||
"http-proxy-middleware": "^3.0.3",
|
||||
"jsonrepair": "^3.14.0",
|
||||
"mysql2": "^3.22.5",
|
||||
"qrcode": "^1.5.4",
|
||||
"redis": "^4.7.1",
|
||||
"sharp": "^0.35.2",
|
||||
"undici": "^6.26.0"
|
||||
}
|
||||
}
|
||||
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(process.cwd(), '.env'));
|
||||
|
||||
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, '');
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
async function fetchText(url, init = {}) {
|
||||
const res = await undiciFetch(url, {
|
||||
...init,
|
||||
dispatcher: url.startsWith('https://127.0.0.1') ? insecureDispatcher : undefined,
|
||||
});
|
||||
const text = await res.text();
|
||||
return { res, text };
|
||||
}
|
||||
|
||||
async function checkJson(pathname) {
|
||||
const { res, text } = await fetchText(`${publicBase}${pathname}`);
|
||||
let json = null;
|
||||
try {
|
||||
json = JSON.parse(text);
|
||||
} catch {
|
||||
json = null;
|
||||
}
|
||||
return { ok: res.ok, status: res.status, json, text: json ? undefined : text.slice(0, 160) };
|
||||
}
|
||||
|
||||
async function checkSseHeaders(pathname) {
|
||||
const { res } = await fetchText(`${publicBase}${pathname}`, {
|
||||
headers: { Accept: 'text/event-stream' },
|
||||
});
|
||||
return {
|
||||
ok: res.status === 401 || res.ok,
|
||||
status: res.status,
|
||||
xAccelBuffering: res.headers.get('x-accel-buffering'),
|
||||
contentType: res.headers.get('content-type'),
|
||||
cacheControl: res.headers.get('cache-control'),
|
||||
};
|
||||
}
|
||||
|
||||
async function checkTarget(target) {
|
||||
const { res, text } = await fetchText(new URL('/status', target).toString());
|
||||
return { target, ok: res.ok, status: res.status, text: text.slice(0, 80) };
|
||||
}
|
||||
|
||||
const result = {
|
||||
ok: true,
|
||||
publicBase,
|
||||
checkedAt: new Date().toISOString(),
|
||||
status: await checkJson('/api/status').catch((err) => ({ ok: false, error: err.message })),
|
||||
runtime: await checkJson('/api/runtime/status').catch((err) => ({ ok: false, error: err.message })),
|
||||
sse: {
|
||||
sessions: await checkSseHeaders('/api/sessions/check-stream-runtime/events').catch((err) => ({
|
||||
ok: false,
|
||||
error: err.message,
|
||||
})),
|
||||
agentRuns: await checkSseHeaders('/api/agent/runs/check-stream-runtime/events').catch((err) => ({
|
||||
ok: false,
|
||||
error: err.message,
|
||||
})),
|
||||
},
|
||||
targets: await Promise.all(targets.map((target) => checkTarget(target).catch((err) => ({
|
||||
target,
|
||||
ok: false,
|
||||
error: err.message,
|
||||
})))),
|
||||
};
|
||||
|
||||
result.ok = Boolean(
|
||||
result.status.ok &&
|
||||
result.runtime.ok &&
|
||||
result.sse.sessions.ok &&
|
||||
result.sse.agentRuns.ok &&
|
||||
result.targets.every((target) => target.ok),
|
||||
);
|
||||
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
process.exit(result.ok ? 0 : 1);
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createClient } from 'redis';
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(path.join(process.cwd(), '.env'));
|
||||
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const configuredWorkers = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.map((_, index) => `goosed-${index + 1}`);
|
||||
const action = process.argv[2] || 'status';
|
||||
const workerId = process.argv[3] || null;
|
||||
|
||||
function usage() {
|
||||
console.error('Usage: node scripts/runtime-worker-drain.mjs <status|drain|undrain> [goosed-N]');
|
||||
}
|
||||
|
||||
function workerKey(id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
}
|
||||
|
||||
if (!['status', 'drain', 'undrain'].includes(action)) {
|
||||
usage();
|
||||
process.exit(2);
|
||||
}
|
||||
if (['drain', 'undrain'].includes(action) && !workerId) {
|
||||
usage();
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const client = createClient({ url: redisUrl });
|
||||
client.on('error', (err) => {
|
||||
console.error(`Redis error: ${err instanceof Error ? err.message : err}`);
|
||||
});
|
||||
await client.connect();
|
||||
|
||||
if (action === 'drain') {
|
||||
await client.set(workerKey(workerId, 'drain'), '1');
|
||||
}
|
||||
if (action === 'undrain') {
|
||||
await client.del(workerKey(workerId, 'drain'));
|
||||
}
|
||||
|
||||
const keys = await client.keys(workerKey('*', 'active_streams'));
|
||||
const workers = [
|
||||
...configuredWorkers,
|
||||
...keys
|
||||
.map((key) => key.split(':').at(-2))
|
||||
.filter(Boolean),
|
||||
];
|
||||
if (workerId && !workers.includes(workerId)) workers.push(workerId);
|
||||
|
||||
const rows = [];
|
||||
for (const id of [...new Set(workers)].sort()) {
|
||||
const values = await client.mGet([
|
||||
workerKey(id, 'active_streams'),
|
||||
workerKey(id, 'drain'),
|
||||
workerKey(id, 'stream_open_count'),
|
||||
workerKey(id, 'stream_abort_count'),
|
||||
workerKey(id, 'stream_error_count'),
|
||||
workerKey(id, 'last_stream_started_at'),
|
||||
workerKey(id, 'last_stream_ended_at'),
|
||||
]);
|
||||
rows.push({
|
||||
id,
|
||||
activeStreams: Number(values[0] || 0),
|
||||
drain: /^(1|true|yes)$/i.test(String(values[1] || '')),
|
||||
streamOpenCount: Number(values[2] || 0),
|
||||
streamAbortCount: Number(values[3] || 0),
|
||||
streamErrorCount: Number(values[4] || 0),
|
||||
lastStreamStartedAt: values[5] ? Number(values[5]) : null,
|
||||
lastStreamEndedAt: values[6] ? Number(values[6]) : null,
|
||||
});
|
||||
}
|
||||
|
||||
await client.quit();
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
action,
|
||||
workerId,
|
||||
namespace,
|
||||
workers: rows,
|
||||
}, null, 2));
|
||||
+45
-23
@@ -8620,6 +8620,7 @@ function createRuntimeRouter({
|
||||
if (/^(1|true|yes)$/i.test(String(values[5] ?? ""))) return Number.POSITIVE_INFINITY;
|
||||
return readNumber(values[0]) * 3 + readNumber(values[1]) + readNumber(values[2]) * 0.01 + readNumber(values[3]) * 5 + readNumber(values[4]) * 2;
|
||||
};
|
||||
const workerScoreFromValues = (values = []) => readNumber(values[0]) * 3 + readNumber(values[1]) + readNumber(values[2]) * 0.01 + readNumber(values[3]) * 5 + readNumber(values[4]) * 2;
|
||||
return {
|
||||
async pickTarget(fallbackPick, orderedTargets = targets) {
|
||||
const client = await getClient();
|
||||
@@ -8650,20 +8651,24 @@ function createRuntimeRouter({
|
||||
if (!client || !target) return;
|
||||
const workerId = workerIdForTarget(target);
|
||||
const streamKey = sessionId ? key("stream", sessionId, "status") : null;
|
||||
const multi = client.multi().incr(key("worker", workerId, "active_streams")).set(key("worker", workerId, "heartbeat"), String(Date.now()), { EX: 30 });
|
||||
const now = String(Date.now());
|
||||
const multi = client.multi().incr(key("worker", workerId, "active_streams")).incr(key("worker", workerId, "stream_open_count")).set(key("worker", workerId, "last_stream_started_at"), now).set(key("worker", workerId, "heartbeat"), now, { EX: 30 });
|
||||
if (streamKey) multi.set(streamKey, "active", { EX: 60 * 60 });
|
||||
await multi.exec().catch(() => null);
|
||||
},
|
||||
async streamEnded(sessionId, target) {
|
||||
async streamEnded(sessionId, target, { status = "closed" } = {}) {
|
||||
const client = await getClient();
|
||||
if (!client || !target) return;
|
||||
const workerId = workerIdForTarget(target);
|
||||
const streamKey = sessionId ? key("stream", sessionId, "status") : null;
|
||||
const activeKey = key("worker", workerId, "active_streams");
|
||||
const now = String(Date.now());
|
||||
const nextValue = await client.decr(activeKey).catch(() => null);
|
||||
const multi = client.multi().set(key("worker", workerId, "heartbeat"), String(Date.now()), { EX: 30 });
|
||||
const multi = client.multi().set(key("worker", workerId, "last_stream_ended_at"), now).set(key("worker", workerId, "heartbeat"), now, { EX: 30 });
|
||||
if (Number(nextValue ?? 0) < 0) multi.set(activeKey, "0");
|
||||
if (streamKey) multi.set(streamKey, "closed", { EX: 600 });
|
||||
if (status === "aborted") multi.incr(key("worker", workerId, "stream_abort_count"));
|
||||
if (status === "error") multi.incr(key("worker", workerId, "stream_error_count"));
|
||||
if (streamKey) multi.set(streamKey, status, { EX: 600 });
|
||||
await multi.exec().catch(() => null);
|
||||
},
|
||||
async getStatus() {
|
||||
@@ -8678,7 +8683,12 @@ function createRuntimeRouter({
|
||||
key("worker", workerId, "error_rate"),
|
||||
key("worker", workerId, "memory_pressure"),
|
||||
key("worker", workerId, "heartbeat"),
|
||||
key("worker", workerId, "drain")
|
||||
key("worker", workerId, "drain"),
|
||||
key("worker", workerId, "stream_open_count"),
|
||||
key("worker", workerId, "stream_abort_count"),
|
||||
key("worker", workerId, "stream_error_count"),
|
||||
key("worker", workerId, "last_stream_started_at"),
|
||||
key("worker", workerId, "last_stream_ended_at")
|
||||
]).catch(() => []) : [];
|
||||
workers.push({
|
||||
id: workerId,
|
||||
@@ -8689,7 +8699,13 @@ function createRuntimeRouter({
|
||||
errorRate: readNumber(values?.[3]),
|
||||
memoryPressure: readNumber(values?.[4]),
|
||||
heartbeat: values?.[5] ? Number(values[5]) : null,
|
||||
drain: /^(1|true|yes)$/i.test(String(values?.[6] ?? ""))
|
||||
drain: /^(1|true|yes)$/i.test(String(values?.[6] ?? "")),
|
||||
streamOpenCount: readNumber(values?.[7]),
|
||||
streamAbortCount: readNumber(values?.[8]),
|
||||
streamErrorCount: readNumber(values?.[9]),
|
||||
lastStreamStartedAt: values?.[10] ? Number(values[10]) : null,
|
||||
lastStreamEndedAt: values?.[11] ? Number(values[11]) : null,
|
||||
score: workerScoreFromValues(values)
|
||||
});
|
||||
}
|
||||
return {
|
||||
@@ -9215,9 +9231,9 @@ function createTkmindProxy({
|
||||
if (!runtimeRouter || !sessionId || !target) return;
|
||||
await runtimeRouter.streamStarted(sessionId, target).catch(() => null);
|
||||
}
|
||||
async function markStreamEnded(sessionId, target) {
|
||||
async function markStreamEnded(sessionId, target, options = {}) {
|
||||
if (!runtimeRouter || !sessionId || !target) return;
|
||||
await runtimeRouter.streamEnded(sessionId, target).catch(() => null);
|
||||
await runtimeRouter.streamEnded(sessionId, target, options).catch(() => null);
|
||||
}
|
||||
async function getRuntimeStatus() {
|
||||
const targetStatuses = [];
|
||||
@@ -9262,8 +9278,8 @@ function createTkmindProxy({
|
||||
if (!session?.id) {
|
||||
throw new Error("\u521B\u5EFA\u4F1A\u8BDD\u5931\u8D25\uFF1A\u7F3A\u5C11 session id");
|
||||
}
|
||||
await userAuth2.registerAgentSession(userId, session.id, startTarget);
|
||||
await rememberSessionTarget(session.id, startTarget);
|
||||
await userAuth2.registerAgentSession(userId, session.id, startTarget);
|
||||
if (resolvedSessionPolicy?.gooseMode) {
|
||||
const modeRes = await apiFetch(startTarget, apiSecret, "/agent/update_session", {
|
||||
method: "POST",
|
||||
@@ -9302,7 +9318,7 @@ function createTkmindProxy({
|
||||
if (targets.length <= 1 || !sessionId) return primaryTarget;
|
||||
try {
|
||||
const routedTarget = await runtimeRouter?.resolveSessionTarget(sessionId);
|
||||
if (routedTarget && targets.includes(routedTarget)) return routedTarget;
|
||||
if (routedTarget) return routedTarget;
|
||||
const { target, node } = await userAuth2.getSessionTarget(sessionId);
|
||||
if (target && targets.includes(target)) return target;
|
||||
return targets[node] ?? primaryTarget;
|
||||
@@ -9493,12 +9509,12 @@ function createTkmindProxy({
|
||||
}
|
||||
const session = JSON.parse(text);
|
||||
if (session?.id) {
|
||||
await rememberSessionTarget(session.id, startTarget);
|
||||
await userAuth2.registerAgentSession(
|
||||
req.currentUser.id,
|
||||
session.id,
|
||||
startTarget
|
||||
);
|
||||
await rememberSessionTarget(session.id, startTarget);
|
||||
if (sessionPolicy.gooseMode) {
|
||||
const modeRes = await apiFetch(startTarget, apiSecret, "/agent/update_session", {
|
||||
method: "POST",
|
||||
@@ -9801,12 +9817,18 @@ function createTkmindProxy({
|
||||
}, 2e4);
|
||||
res.on("drain", () => source.resume());
|
||||
await markStreamStarted(sessionId, sessionTarget);
|
||||
let streamCloseStatus = "closed";
|
||||
try {
|
||||
await pipeline(source, linkSanitizer, billingTransform, clientSink);
|
||||
try {
|
||||
await pipeline(source, linkSanitizer, billingTransform, clientSink);
|
||||
} catch (err) {
|
||||
streamCloseStatus = clientClosed || upstreamAbort.signal.aborted ? "aborted" : "error";
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
clearInterval(keepalive);
|
||||
req.off("close", abortUpstream);
|
||||
await markStreamEnded(sessionId, sessionTarget);
|
||||
await markStreamEnded(sessionId, sessionTarget, { status: streamCloseStatus });
|
||||
if (!res.writableEnded) res.end();
|
||||
}
|
||||
} catch (err) {
|
||||
@@ -9949,9 +9971,9 @@ function createTkmindProxy({
|
||||
sessionScoped,
|
||||
proxyFallback,
|
||||
proxySessionEvents,
|
||||
getRuntimeStatus,
|
||||
resolveTarget,
|
||||
startSessionForUser,
|
||||
getRuntimeStatus,
|
||||
submitSessionReplyForUser,
|
||||
apiFetch: async (pathname, init) => apiFetch(await pickTarget(), apiSecret, pathname, init),
|
||||
apiFetchTo: (target, pathname, init) => apiFetch(target, apiSecret, pathname, init)
|
||||
@@ -14764,7 +14786,7 @@ function buildOutputProfile(mimeType) {
|
||||
mimeType: "image/png",
|
||||
baseQuality: 90,
|
||||
minQuality: 70,
|
||||
encode: (pipeline, quality) => pipeline.png({
|
||||
encode: (pipeline2, quality) => pipeline2.png({
|
||||
compressionLevel: 9,
|
||||
adaptiveFiltering: true,
|
||||
palette: true,
|
||||
@@ -14779,7 +14801,7 @@ function buildOutputProfile(mimeType) {
|
||||
mimeType: "image/webp",
|
||||
baseQuality: 84,
|
||||
minQuality: WEBP_MIN_QUALITY,
|
||||
encode: (pipeline, quality) => pipeline.webp({
|
||||
encode: (pipeline2, quality) => pipeline2.webp({
|
||||
quality,
|
||||
alphaQuality: Math.min(100, quality + 8),
|
||||
effort: 5
|
||||
@@ -14791,7 +14813,7 @@ function buildOutputProfile(mimeType) {
|
||||
mimeType: "image/jpeg",
|
||||
baseQuality: 84,
|
||||
minQuality: JPEG_MIN_QUALITY,
|
||||
encode: (pipeline, quality) => pipeline.jpeg({
|
||||
encode: (pipeline2, quality) => pipeline2.jpeg({
|
||||
quality,
|
||||
mozjpeg: true,
|
||||
chromaSubsampling: "4:4:4"
|
||||
@@ -14803,7 +14825,7 @@ function deriveOutputFilename(filename, extension) {
|
||||
return `${raw}${extension}`;
|
||||
}
|
||||
async function renderVariant(buffer, metadata, profile, { width, height, quality, maxPixels }) {
|
||||
const pipeline = sharp(buffer, {
|
||||
const pipeline2 = sharp(buffer, {
|
||||
failOn: "error",
|
||||
limitInputPixels: maxPixels,
|
||||
sequentialRead: true
|
||||
@@ -14813,7 +14835,7 @@ async function renderVariant(buffer, metadata, profile, { width, height, quality
|
||||
fit: "inside",
|
||||
withoutEnlargement: true
|
||||
});
|
||||
const encoded = await profile.encode(pipeline, quality).toBuffer();
|
||||
const encoded = await profile.encode(pipeline2, quality).toBuffer();
|
||||
return encoded;
|
||||
}
|
||||
async function normalizeImageForStorage({
|
||||
@@ -35854,20 +35876,20 @@ api.get("/status", async (_req, res, next) => {
|
||||
});
|
||||
api.get("/runtime/status", async (_req, res) => {
|
||||
await userAuthReady;
|
||||
if (!tkmindProxy) {
|
||||
return res.status(503).json({ ok: false, message: "\u4F1A\u8BDD\u4EE3\u7406\u5C1A\u672A\u5C31\u7EEA" });
|
||||
if (!tkmindProxy?.getRuntimeStatus) {
|
||||
return res.status(503).json({ ok: false, message: "runtime router unavailable" });
|
||||
}
|
||||
try {
|
||||
const status = await tkmindProxy.getRuntimeStatus();
|
||||
return res.json({
|
||||
ok: true,
|
||||
at: Date.now(),
|
||||
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
||||
...status
|
||||
});
|
||||
} catch (err) {
|
||||
return res.status(502).json({
|
||||
ok: false,
|
||||
message: err instanceof Error ? err.message : "\u8BFB\u53D6 runtime \u72B6\u6001\u5931\u8D25"
|
||||
message: err instanceof Error ? err.message : "runtime status failed"
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user