chore: add runtime metrics agent and slo report
This commit is contained in:
John
@@ -279,6 +279,14 @@ async function writeMetadata() {
|
||||
path.join(root, 'scripts', 'runtime-worker-metrics.mjs'),
|
||||
path.join(runtimeRoot, 'scripts', 'runtime-worker-metrics.mjs'),
|
||||
);
|
||||
await fs.copyFile(
|
||||
path.join(root, 'scripts', 'runtime-slo-report.mjs'),
|
||||
path.join(runtimeRoot, 'scripts', 'runtime-slo-report.mjs'),
|
||||
);
|
||||
await fs.copyFile(
|
||||
path.join(root, 'scripts', 'install-runtime-metrics-agent.sh'),
|
||||
path.join(runtimeRoot, 'scripts', 'install-runtime-metrics-agent.sh'),
|
||||
);
|
||||
await fs.copyFile(
|
||||
path.join(root, 'scripts', 'check-tool-runtime.mjs'),
|
||||
path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'),
|
||||
@@ -332,6 +340,8 @@ async function writeMetadata() {
|
||||
' node scripts/runtime-worker-drain.mjs reconcile --apply',
|
||||
' node scripts/runtime-worker-metrics.mjs status',
|
||||
' node scripts/runtime-worker-metrics.mjs sample',
|
||||
' bash scripts/install-runtime-metrics-agent.sh',
|
||||
' node scripts/runtime-slo-report.mjs',
|
||||
' node scripts/runtime-worker-drain.mjs drain goosed-3',
|
||||
' node scripts/runtime-worker-drain.mjs undrain goosed-3',
|
||||
' node scripts/check-tool-runtime.mjs',
|
||||
@@ -355,6 +365,9 @@ async function main() {
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'wechat-mp-menu.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-stream-runtime.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-drain.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-metrics.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-slo-report.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-runtime-metrics-agent.sh'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
|
||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
|
||||
console.log('');
|
||||
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||
SCRIPT="${MEMIND_RUNTIME_METRICS_SCRIPT:-$ROOT/scripts/runtime-worker-metrics.mjs}"
|
||||
LABEL="${MEMIND_RUNTIME_METRICS_LABEL:-cn.tkmind.memind-runtime-metrics}"
|
||||
PLIST="$HOME/Library/LaunchAgents/${LABEL}.plist"
|
||||
LOG="$HOME/Library/Logs/memind-runtime-metrics.log"
|
||||
GUI="gui/$(id -u)"
|
||||
INTERVAL="${MEMIND_RUNTIME_METRICS_INTERVAL:-60}"
|
||||
|
||||
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
|
||||
|
||||
if [[ ! -x "$NODE_BIN" ]]; then
|
||||
NODE_BIN="$(command -v node)"
|
||||
fi
|
||||
if [[ ! -f "$SCRIPT" ]]; then
|
||||
echo "runtime metrics script not found: $SCRIPT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cat > "$PLIST" <<EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>Label</key>
|
||||
<string>$LABEL</string>
|
||||
<key>ProgramArguments</key>
|
||||
<array>
|
||||
<string>$NODE_BIN</string>
|
||||
<string>$SCRIPT</string>
|
||||
<string>sample</string>
|
||||
</array>
|
||||
<key>WorkingDirectory</key>
|
||||
<string>$ROOT</string>
|
||||
<key>StartInterval</key>
|
||||
<integer>$INTERVAL</integer>
|
||||
<key>RunAtLoad</key>
|
||||
<true/>
|
||||
<key>StandardOutPath</key>
|
||||
<string>$LOG</string>
|
||||
<key>StandardErrorPath</key>
|
||||
<string>$LOG</string>
|
||||
<key>EnvironmentVariables</key>
|
||||
<dict>
|
||||
<key>PATH</key>
|
||||
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||
</dict>
|
||||
</dict>
|
||||
</plist>
|
||||
EOF
|
||||
|
||||
plutil -lint "$PLIST"
|
||||
launchctl bootout "$GUI/$LABEL" 2>/dev/null || true
|
||||
launchctl bootstrap "$GUI" "$PLIST"
|
||||
launchctl enable "$GUI/$LABEL"
|
||||
launchctl kickstart -k "$GUI/$LABEL"
|
||||
|
||||
echo "installed $PLIST"
|
||||
echo "script: $SCRIPT"
|
||||
echo "interval: ${INTERVAL}s"
|
||||
echo "log: $LOG"
|
||||
Executable
+239
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env node
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { Agent, fetch as undiciFetch } from 'undici';
|
||||
import mysql from 'mysql2/promise';
|
||||
import { createClient } from 'redis';
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||
const idx = trimmed.indexOf('=');
|
||||
if (idx < 0) continue;
|
||||
const key = trimmed.slice(0, idx).trim();
|
||||
const value = trimmed.slice(idx + 1).trim();
|
||||
if (!process.env[key]) process.env[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
function parseMysqlConfig() {
|
||||
if (process.env.DATABASE_URL) {
|
||||
const url = new URL(process.env.DATABASE_URL);
|
||||
if (url.protocol !== 'mysql:') {
|
||||
throw new Error(`Unsupported DATABASE_URL scheme for SLO report: ${url.protocol}`);
|
||||
}
|
||||
return {
|
||||
host: url.hostname,
|
||||
port: Number(url.port || 3306),
|
||||
user: decodeURIComponent(url.username),
|
||||
password: decodeURIComponent(url.password),
|
||||
database: url.pathname.replace(/^\/+/, ''),
|
||||
charset: 'utf8mb4',
|
||||
};
|
||||
}
|
||||
return {
|
||||
host: process.env.MYSQL_HOST,
|
||||
port: Number(process.env.MYSQL_PORT || 3306),
|
||||
user: process.env.MYSQL_USER,
|
||||
password: process.env.MYSQL_PASSWORD,
|
||||
database: process.env.MYSQL_DATABASE,
|
||||
charset: 'utf8mb4',
|
||||
};
|
||||
}
|
||||
|
||||
function walkCount(root) {
|
||||
const stack = [root];
|
||||
let files = 0;
|
||||
let dirs = 0;
|
||||
let bytes = 0;
|
||||
while (stack.length) {
|
||||
const current = stack.pop();
|
||||
let entries = [];
|
||||
try {
|
||||
entries = fs.readdirSync(current, { withFileTypes: true });
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const full = path.join(current, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
dirs += 1;
|
||||
stack.push(full);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile()) continue;
|
||||
files += 1;
|
||||
try {
|
||||
bytes += fs.statSync(full).size;
|
||||
} catch {
|
||||
// ignore files that disappear while counting
|
||||
}
|
||||
}
|
||||
}
|
||||
return { files, dirs, bytes };
|
||||
}
|
||||
|
||||
function uniquePaths(paths) {
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
for (const item of paths) {
|
||||
const resolved = path.resolve(item);
|
||||
if (seen.has(resolved)) continue;
|
||||
seen.add(resolved);
|
||||
result.push(resolved);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function tableCountQueries() {
|
||||
return [
|
||||
['h5_users', 'users'],
|
||||
['h5_user_sessions', 'sessions'],
|
||||
['h5_agent_runs', 'agentRuns'],
|
||||
['h5_agent_run_events', 'agentRunEvents'],
|
||||
['h5_capability_grants', 'capabilityGrants'],
|
||||
['h5_llm_provider_keys', 'llmProviderKeys'],
|
||||
['mindspace_assets', 'mindspaceAssets'],
|
||||
['mindspace_pages', 'mindspacePages'],
|
||||
['mindspace_publications', 'mindspacePublications'],
|
||||
];
|
||||
}
|
||||
|
||||
async function safeTableCount(conn, table) {
|
||||
try {
|
||||
const [rows] = await conn.query(`SELECT COUNT(*) AS count FROM \`${table}\``);
|
||||
return Number(rows[0]?.count ?? 0);
|
||||
} catch (err) {
|
||||
return { error: err instanceof Error ? err.message : String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
async function readDbSummary() {
|
||||
const conn = await mysql.createConnection(parseMysqlConfig());
|
||||
try {
|
||||
const result = {};
|
||||
for (const [table, key] of tableCountQueries()) {
|
||||
result[key] = await safeTableCount(conn, table);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
await conn.end();
|
||||
}
|
||||
}
|
||||
|
||||
async function readRedisSummary(namespace, redisUrl) {
|
||||
const client = createClient({ url: redisUrl });
|
||||
await client.connect();
|
||||
try {
|
||||
const workerKeys = await client.keys(`${namespace}:worker:*:active_streams`);
|
||||
const streamKeys = await client.keys(`${namespace}:stream:*:status`);
|
||||
const sessionKeys = await client.keys(`${namespace}:session:*:target`);
|
||||
return {
|
||||
workerCount: workerKeys.length,
|
||||
streamStatusCount: streamKeys.length,
|
||||
sessionTargetCount: sessionKeys.length,
|
||||
};
|
||||
} finally {
|
||||
await client.quit();
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeRuntime(runtimeJson) {
|
||||
const workers = runtimeJson?.router?.workers ?? [];
|
||||
const staleMetricMs = 2 * 60 * 1000;
|
||||
const now = Date.now();
|
||||
return {
|
||||
routerEnabled: Boolean(runtimeJson?.router?.enabled),
|
||||
publicBaseUrl: runtimeJson?.publicBaseUrl ?? null,
|
||||
toolRuntime: runtimeJson?.toolRuntime ?? null,
|
||||
workers: workers.map((worker) => ({
|
||||
id: worker.id,
|
||||
healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null,
|
||||
activeStreams: worker.activeStreams,
|
||||
streamOpenCount: worker.streamOpenCount,
|
||||
streamAbortCount: worker.streamAbortCount,
|
||||
streamErrorCount: worker.streamErrorCount,
|
||||
cpuLoad: worker.cpuLoad,
|
||||
memoryPressure: worker.memoryPressure,
|
||||
fdPressure: worker.fdPressure,
|
||||
fdCount: worker.fdCount,
|
||||
containerHealth: worker.containerHealth,
|
||||
metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null,
|
||||
metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false,
|
||||
score: worker.score,
|
||||
drain: worker.drain,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
const envFile = process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env');
|
||||
const appRoot = process.env.MEMIND_APP_ROOT || path.dirname(path.resolve(envFile));
|
||||
loadEnvFile(envFile);
|
||||
|
||||
const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, '');
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const mindSpaceRoots = uniquePaths([
|
||||
path.join(appRoot, 'MindSpace'),
|
||||
process.env.MINDSPACE_STORAGE_ROOT || path.join(appRoot, 'data', 'mindspace'),
|
||||
]);
|
||||
const dispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||
|
||||
async function fetchRuntime() {
|
||||
const res = await undiciFetch(`${publicBase}/api/runtime/status`, {
|
||||
dispatcher: publicBase.startsWith('https://127.0.0.1') ? dispatcher : undefined,
|
||||
});
|
||||
const text = await res.text();
|
||||
return {
|
||||
ok: res.ok,
|
||||
status: res.status,
|
||||
json: JSON.parse(text),
|
||||
};
|
||||
}
|
||||
|
||||
const runtime = await fetchRuntime().catch((err) => ({ ok: false, error: err.message }));
|
||||
const runtimeSummary = runtime.ok ? summarizeRuntime(runtime.json) : null;
|
||||
const db = await readDbSummary().catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
|
||||
const redis = await readRedisSummary(namespace, redisUrl).catch((err) => ({
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}));
|
||||
const mindSpace = mindSpaceRoots.map((root) => (
|
||||
fs.existsSync(root)
|
||||
? { root, ...walkCount(root) }
|
||||
: { root, error: 'missing' }
|
||||
));
|
||||
|
||||
const failures = [];
|
||||
if (!runtime.ok) failures.push('runtime_status_unavailable');
|
||||
if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled');
|
||||
for (const worker of runtimeSummary?.workers ?? []) {
|
||||
if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`);
|
||||
if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`);
|
||||
if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`);
|
||||
if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`);
|
||||
}
|
||||
if (runtimeSummary?.toolRuntime?.chatInjectsCodeTools !== false) failures.push('chat_injects_code_tools');
|
||||
|
||||
const report = {
|
||||
ok: failures.length === 0,
|
||||
checkedAt: new Date().toISOString(),
|
||||
publicBase,
|
||||
appRoot,
|
||||
envFile,
|
||||
namespace,
|
||||
runtime: runtimeSummary,
|
||||
db,
|
||||
redis,
|
||||
mindSpace,
|
||||
failures,
|
||||
writes: {
|
||||
database: false,
|
||||
mindSpace: false,
|
||||
redis: false,
|
||||
},
|
||||
};
|
||||
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
process.exit(report.ok ? 0 : 1);
|
||||
Reference in New Issue
Block a user