chore: add runtime metrics agent and slo report
This commit is contained in:
@@ -40,6 +40,8 @@ Streaming runtime operations:
|
|||||||
node scripts/runtime-worker-drain.mjs reconcile --apply
|
node scripts/runtime-worker-drain.mjs reconcile --apply
|
||||||
node scripts/runtime-worker-metrics.mjs status
|
node scripts/runtime-worker-metrics.mjs status
|
||||||
node scripts/runtime-worker-metrics.mjs sample
|
node scripts/runtime-worker-metrics.mjs sample
|
||||||
|
bash scripts/install-runtime-metrics-agent.sh
|
||||||
|
node scripts/runtime-slo-report.mjs
|
||||||
node scripts/runtime-worker-drain.mjs drain goosed-3
|
node scripts/runtime-worker-drain.mjs drain goosed-3
|
||||||
node scripts/runtime-worker-drain.mjs undrain goosed-3
|
node scripts/runtime-worker-drain.mjs undrain goosed-3
|
||||||
node scripts/check-tool-runtime.mjs
|
node scripts/check-tool-runtime.mjs
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||||
|
SCRIPT="${MEMIND_RUNTIME_METRICS_SCRIPT:-$ROOT/scripts/runtime-worker-metrics.mjs}"
|
||||||
|
LABEL="${MEMIND_RUNTIME_METRICS_LABEL:-cn.tkmind.memind-runtime-metrics}"
|
||||||
|
PLIST="$HOME/Library/LaunchAgents/${LABEL}.plist"
|
||||||
|
LOG="$HOME/Library/Logs/memind-runtime-metrics.log"
|
||||||
|
GUI="gui/$(id -u)"
|
||||||
|
INTERVAL="${MEMIND_RUNTIME_METRICS_INTERVAL:-60}"
|
||||||
|
|
||||||
|
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
|
||||||
|
|
||||||
|
if [[ ! -x "$NODE_BIN" ]]; then
|
||||||
|
NODE_BIN="$(command -v node)"
|
||||||
|
fi
|
||||||
|
if [[ ! -f "$SCRIPT" ]]; then
|
||||||
|
echo "runtime metrics script not found: $SCRIPT" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat > "$PLIST" <<EOF
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||||
|
<plist version="1.0">
|
||||||
|
<dict>
|
||||||
|
<key>Label</key>
|
||||||
|
<string>$LABEL</string>
|
||||||
|
<key>ProgramArguments</key>
|
||||||
|
<array>
|
||||||
|
<string>$NODE_BIN</string>
|
||||||
|
<string>$SCRIPT</string>
|
||||||
|
<string>sample</string>
|
||||||
|
</array>
|
||||||
|
<key>WorkingDirectory</key>
|
||||||
|
<string>$ROOT</string>
|
||||||
|
<key>StartInterval</key>
|
||||||
|
<integer>$INTERVAL</integer>
|
||||||
|
<key>RunAtLoad</key>
|
||||||
|
<true/>
|
||||||
|
<key>StandardOutPath</key>
|
||||||
|
<string>$LOG</string>
|
||||||
|
<key>StandardErrorPath</key>
|
||||||
|
<string>$LOG</string>
|
||||||
|
<key>EnvironmentVariables</key>
|
||||||
|
<dict>
|
||||||
|
<key>PATH</key>
|
||||||
|
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||||
|
</dict>
|
||||||
|
</dict>
|
||||||
|
</plist>
|
||||||
|
EOF
|
||||||
|
|
||||||
|
plutil -lint "$PLIST"
|
||||||
|
launchctl bootout "$GUI/$LABEL" 2>/dev/null || true
|
||||||
|
launchctl bootstrap "$GUI" "$PLIST"
|
||||||
|
launchctl enable "$GUI/$LABEL"
|
||||||
|
launchctl kickstart -k "$GUI/$LABEL"
|
||||||
|
|
||||||
|
echo "installed $PLIST"
|
||||||
|
echo "script: $SCRIPT"
|
||||||
|
echo "interval: ${INTERVAL}s"
|
||||||
|
echo "log: $LOG"
|
||||||
+239
@@ -0,0 +1,239 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { Agent, fetch as undiciFetch } from 'undici';
|
||||||
|
import mysql from 'mysql2/promise';
|
||||||
|
import { createClient } from 'redis';
|
||||||
|
|
||||||
|
function loadEnvFile(filePath) {
|
||||||
|
if (!fs.existsSync(filePath)) return;
|
||||||
|
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||||
|
const idx = trimmed.indexOf('=');
|
||||||
|
if (idx < 0) continue;
|
||||||
|
const key = trimmed.slice(0, idx).trim();
|
||||||
|
const value = trimmed.slice(idx + 1).trim();
|
||||||
|
if (!process.env[key]) process.env[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseMysqlConfig() {
|
||||||
|
if (process.env.DATABASE_URL) {
|
||||||
|
const url = new URL(process.env.DATABASE_URL);
|
||||||
|
if (url.protocol !== 'mysql:') {
|
||||||
|
throw new Error(`Unsupported DATABASE_URL scheme for SLO report: ${url.protocol}`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
host: url.hostname,
|
||||||
|
port: Number(url.port || 3306),
|
||||||
|
user: decodeURIComponent(url.username),
|
||||||
|
password: decodeURIComponent(url.password),
|
||||||
|
database: url.pathname.replace(/^\/+/, ''),
|
||||||
|
charset: 'utf8mb4',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
host: process.env.MYSQL_HOST,
|
||||||
|
port: Number(process.env.MYSQL_PORT || 3306),
|
||||||
|
user: process.env.MYSQL_USER,
|
||||||
|
password: process.env.MYSQL_PASSWORD,
|
||||||
|
database: process.env.MYSQL_DATABASE,
|
||||||
|
charset: 'utf8mb4',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkCount(root) {
|
||||||
|
const stack = [root];
|
||||||
|
let files = 0;
|
||||||
|
let dirs = 0;
|
||||||
|
let bytes = 0;
|
||||||
|
while (stack.length) {
|
||||||
|
const current = stack.pop();
|
||||||
|
let entries = [];
|
||||||
|
try {
|
||||||
|
entries = fs.readdirSync(current, { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (const entry of entries) {
|
||||||
|
const full = path.join(current, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
dirs += 1;
|
||||||
|
stack.push(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!entry.isFile()) continue;
|
||||||
|
files += 1;
|
||||||
|
try {
|
||||||
|
bytes += fs.statSync(full).size;
|
||||||
|
} catch {
|
||||||
|
// ignore files that disappear while counting
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { files, dirs, bytes };
|
||||||
|
}
|
||||||
|
|
||||||
|
function uniquePaths(paths) {
|
||||||
|
const seen = new Set();
|
||||||
|
const result = [];
|
||||||
|
for (const item of paths) {
|
||||||
|
const resolved = path.resolve(item);
|
||||||
|
if (seen.has(resolved)) continue;
|
||||||
|
seen.add(resolved);
|
||||||
|
result.push(resolved);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tableCountQueries() {
|
||||||
|
return [
|
||||||
|
['h5_users', 'users'],
|
||||||
|
['h5_user_sessions', 'sessions'],
|
||||||
|
['h5_agent_runs', 'agentRuns'],
|
||||||
|
['h5_agent_run_events', 'agentRunEvents'],
|
||||||
|
['h5_capability_grants', 'capabilityGrants'],
|
||||||
|
['h5_llm_provider_keys', 'llmProviderKeys'],
|
||||||
|
['mindspace_assets', 'mindspaceAssets'],
|
||||||
|
['mindspace_pages', 'mindspacePages'],
|
||||||
|
['mindspace_publications', 'mindspacePublications'],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function safeTableCount(conn, table) {
|
||||||
|
try {
|
||||||
|
const [rows] = await conn.query(`SELECT COUNT(*) AS count FROM \`${table}\``);
|
||||||
|
return Number(rows[0]?.count ?? 0);
|
||||||
|
} catch (err) {
|
||||||
|
return { error: err instanceof Error ? err.message : String(err) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readDbSummary() {
|
||||||
|
const conn = await mysql.createConnection(parseMysqlConfig());
|
||||||
|
try {
|
||||||
|
const result = {};
|
||||||
|
for (const [table, key] of tableCountQueries()) {
|
||||||
|
result[key] = await safeTableCount(conn, table);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
await conn.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readRedisSummary(namespace, redisUrl) {
|
||||||
|
const client = createClient({ url: redisUrl });
|
||||||
|
await client.connect();
|
||||||
|
try {
|
||||||
|
const workerKeys = await client.keys(`${namespace}:worker:*:active_streams`);
|
||||||
|
const streamKeys = await client.keys(`${namespace}:stream:*:status`);
|
||||||
|
const sessionKeys = await client.keys(`${namespace}:session:*:target`);
|
||||||
|
return {
|
||||||
|
workerCount: workerKeys.length,
|
||||||
|
streamStatusCount: streamKeys.length,
|
||||||
|
sessionTargetCount: sessionKeys.length,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
await client.quit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarizeRuntime(runtimeJson) {
|
||||||
|
const workers = runtimeJson?.router?.workers ?? [];
|
||||||
|
const staleMetricMs = 2 * 60 * 1000;
|
||||||
|
const now = Date.now();
|
||||||
|
return {
|
||||||
|
routerEnabled: Boolean(runtimeJson?.router?.enabled),
|
||||||
|
publicBaseUrl: runtimeJson?.publicBaseUrl ?? null,
|
||||||
|
toolRuntime: runtimeJson?.toolRuntime ?? null,
|
||||||
|
workers: workers.map((worker) => ({
|
||||||
|
id: worker.id,
|
||||||
|
healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null,
|
||||||
|
activeStreams: worker.activeStreams,
|
||||||
|
streamOpenCount: worker.streamOpenCount,
|
||||||
|
streamAbortCount: worker.streamAbortCount,
|
||||||
|
streamErrorCount: worker.streamErrorCount,
|
||||||
|
cpuLoad: worker.cpuLoad,
|
||||||
|
memoryPressure: worker.memoryPressure,
|
||||||
|
fdPressure: worker.fdPressure,
|
||||||
|
fdCount: worker.fdCount,
|
||||||
|
containerHealth: worker.containerHealth,
|
||||||
|
metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null,
|
||||||
|
metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false,
|
||||||
|
score: worker.score,
|
||||||
|
drain: worker.drain,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const envFile = process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env');
|
||||||
|
const appRoot = process.env.MEMIND_APP_ROOT || path.dirname(path.resolve(envFile));
|
||||||
|
loadEnvFile(envFile);
|
||||||
|
|
||||||
|
const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, '');
|
||||||
|
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||||
|
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||||
|
const mindSpaceRoots = uniquePaths([
|
||||||
|
path.join(appRoot, 'MindSpace'),
|
||||||
|
process.env.MINDSPACE_STORAGE_ROOT || path.join(appRoot, 'data', 'mindspace'),
|
||||||
|
]);
|
||||||
|
const dispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||||
|
|
||||||
|
async function fetchRuntime() {
|
||||||
|
const res = await undiciFetch(`${publicBase}/api/runtime/status`, {
|
||||||
|
dispatcher: publicBase.startsWith('https://127.0.0.1') ? dispatcher : undefined,
|
||||||
|
});
|
||||||
|
const text = await res.text();
|
||||||
|
return {
|
||||||
|
ok: res.ok,
|
||||||
|
status: res.status,
|
||||||
|
json: JSON.parse(text),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const runtime = await fetchRuntime().catch((err) => ({ ok: false, error: err.message }));
|
||||||
|
const runtimeSummary = runtime.ok ? summarizeRuntime(runtime.json) : null;
|
||||||
|
const db = await readDbSummary().catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
|
||||||
|
const redis = await readRedisSummary(namespace, redisUrl).catch((err) => ({
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
}));
|
||||||
|
const mindSpace = mindSpaceRoots.map((root) => (
|
||||||
|
fs.existsSync(root)
|
||||||
|
? { root, ...walkCount(root) }
|
||||||
|
: { root, error: 'missing' }
|
||||||
|
));
|
||||||
|
|
||||||
|
const failures = [];
|
||||||
|
if (!runtime.ok) failures.push('runtime_status_unavailable');
|
||||||
|
if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled');
|
||||||
|
for (const worker of runtimeSummary?.workers ?? []) {
|
||||||
|
if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`);
|
||||||
|
if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`);
|
||||||
|
if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`);
|
||||||
|
if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`);
|
||||||
|
}
|
||||||
|
if (runtimeSummary?.toolRuntime?.chatInjectsCodeTools !== false) failures.push('chat_injects_code_tools');
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
ok: failures.length === 0,
|
||||||
|
checkedAt: new Date().toISOString(),
|
||||||
|
publicBase,
|
||||||
|
appRoot,
|
||||||
|
envFile,
|
||||||
|
namespace,
|
||||||
|
runtime: runtimeSummary,
|
||||||
|
db,
|
||||||
|
redis,
|
||||||
|
mindSpace,
|
||||||
|
failures,
|
||||||
|
writes: {
|
||||||
|
database: false,
|
||||||
|
mindSpace: false,
|
||||||
|
redis: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
process.exit(report.ok ? 0 : 1);
|
||||||
Regular → Executable
@@ -279,6 +279,14 @@ async function writeMetadata() {
|
|||||||
path.join(root, 'scripts', 'runtime-worker-metrics.mjs'),
|
path.join(root, 'scripts', 'runtime-worker-metrics.mjs'),
|
||||||
path.join(runtimeRoot, 'scripts', 'runtime-worker-metrics.mjs'),
|
path.join(runtimeRoot, 'scripts', 'runtime-worker-metrics.mjs'),
|
||||||
);
|
);
|
||||||
|
await fs.copyFile(
|
||||||
|
path.join(root, 'scripts', 'runtime-slo-report.mjs'),
|
||||||
|
path.join(runtimeRoot, 'scripts', 'runtime-slo-report.mjs'),
|
||||||
|
);
|
||||||
|
await fs.copyFile(
|
||||||
|
path.join(root, 'scripts', 'install-runtime-metrics-agent.sh'),
|
||||||
|
path.join(runtimeRoot, 'scripts', 'install-runtime-metrics-agent.sh'),
|
||||||
|
);
|
||||||
await fs.copyFile(
|
await fs.copyFile(
|
||||||
path.join(root, 'scripts', 'check-tool-runtime.mjs'),
|
path.join(root, 'scripts', 'check-tool-runtime.mjs'),
|
||||||
path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'),
|
path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'),
|
||||||
@@ -332,6 +340,8 @@ async function writeMetadata() {
|
|||||||
' node scripts/runtime-worker-drain.mjs reconcile --apply',
|
' node scripts/runtime-worker-drain.mjs reconcile --apply',
|
||||||
' node scripts/runtime-worker-metrics.mjs status',
|
' node scripts/runtime-worker-metrics.mjs status',
|
||||||
' node scripts/runtime-worker-metrics.mjs sample',
|
' node scripts/runtime-worker-metrics.mjs sample',
|
||||||
|
' bash scripts/install-runtime-metrics-agent.sh',
|
||||||
|
' node scripts/runtime-slo-report.mjs',
|
||||||
' node scripts/runtime-worker-drain.mjs drain goosed-3',
|
' node scripts/runtime-worker-drain.mjs drain goosed-3',
|
||||||
' node scripts/runtime-worker-drain.mjs undrain goosed-3',
|
' node scripts/runtime-worker-drain.mjs undrain goosed-3',
|
||||||
' node scripts/check-tool-runtime.mjs',
|
' node scripts/check-tool-runtime.mjs',
|
||||||
@@ -355,6 +365,9 @@ async function main() {
|
|||||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'wechat-mp-menu.mjs'), 0o755);
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'wechat-mp-menu.mjs'), 0o755);
|
||||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-stream-runtime.mjs'), 0o755);
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-stream-runtime.mjs'), 0o755);
|
||||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-drain.mjs'), 0o755);
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-drain.mjs'), 0o755);
|
||||||
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-worker-metrics.mjs'), 0o755);
|
||||||
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'runtime-slo-report.mjs'), 0o755);
|
||||||
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'install-runtime-metrics-agent.sh'), 0o755);
|
||||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'check-tool-runtime.mjs'), 0o755);
|
||||||
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
|
await fs.chmod(path.join(runtimeRoot, 'scripts', 'memind-portal-tunnel.sh'), 0o755);
|
||||||
console.log('');
|
console.log('');
|
||||||
|
|||||||
Executable
+64
@@ -0,0 +1,64 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||||
|
NODE_BIN="${NODE_BIN:-/opt/homebrew/opt/node@24/bin/node}"
|
||||||
|
SCRIPT="${MEMIND_RUNTIME_METRICS_SCRIPT:-$ROOT/scripts/runtime-worker-metrics.mjs}"
|
||||||
|
LABEL="${MEMIND_RUNTIME_METRICS_LABEL:-cn.tkmind.memind-runtime-metrics}"
|
||||||
|
PLIST="$HOME/Library/LaunchAgents/${LABEL}.plist"
|
||||||
|
LOG="$HOME/Library/Logs/memind-runtime-metrics.log"
|
||||||
|
GUI="gui/$(id -u)"
|
||||||
|
INTERVAL="${MEMIND_RUNTIME_METRICS_INTERVAL:-60}"
|
||||||
|
|
||||||
|
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
|
||||||
|
|
||||||
|
if [[ ! -x "$NODE_BIN" ]]; then
|
||||||
|
NODE_BIN="$(command -v node)"
|
||||||
|
fi
|
||||||
|
if [[ ! -f "$SCRIPT" ]]; then
|
||||||
|
echo "runtime metrics script not found: $SCRIPT" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat > "$PLIST" <<EOF
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||||
|
<plist version="1.0">
|
||||||
|
<dict>
|
||||||
|
<key>Label</key>
|
||||||
|
<string>$LABEL</string>
|
||||||
|
<key>ProgramArguments</key>
|
||||||
|
<array>
|
||||||
|
<string>$NODE_BIN</string>
|
||||||
|
<string>$SCRIPT</string>
|
||||||
|
<string>sample</string>
|
||||||
|
</array>
|
||||||
|
<key>WorkingDirectory</key>
|
||||||
|
<string>$ROOT</string>
|
||||||
|
<key>StartInterval</key>
|
||||||
|
<integer>$INTERVAL</integer>
|
||||||
|
<key>RunAtLoad</key>
|
||||||
|
<true/>
|
||||||
|
<key>StandardOutPath</key>
|
||||||
|
<string>$LOG</string>
|
||||||
|
<key>StandardErrorPath</key>
|
||||||
|
<string>$LOG</string>
|
||||||
|
<key>EnvironmentVariables</key>
|
||||||
|
<dict>
|
||||||
|
<key>PATH</key>
|
||||||
|
<string>/opt/homebrew/bin:/opt/homebrew/opt/node@24/bin:/usr/local/bin:/usr/bin:/bin</string>
|
||||||
|
</dict>
|
||||||
|
</dict>
|
||||||
|
</plist>
|
||||||
|
EOF
|
||||||
|
|
||||||
|
plutil -lint "$PLIST"
|
||||||
|
launchctl bootout "$GUI/$LABEL" 2>/dev/null || true
|
||||||
|
launchctl bootstrap "$GUI" "$PLIST"
|
||||||
|
launchctl enable "$GUI/$LABEL"
|
||||||
|
launchctl kickstart -k "$GUI/$LABEL"
|
||||||
|
|
||||||
|
echo "installed $PLIST"
|
||||||
|
echo "script: $SCRIPT"
|
||||||
|
echo "interval: ${INTERVAL}s"
|
||||||
|
echo "log: $LOG"
|
||||||
Executable
+239
@@ -0,0 +1,239 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { Agent, fetch as undiciFetch } from 'undici';
|
||||||
|
import mysql from 'mysql2/promise';
|
||||||
|
import { createClient } from 'redis';
|
||||||
|
|
||||||
|
function loadEnvFile(filePath) {
|
||||||
|
if (!fs.existsSync(filePath)) return;
|
||||||
|
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
||||||
|
const idx = trimmed.indexOf('=');
|
||||||
|
if (idx < 0) continue;
|
||||||
|
const key = trimmed.slice(0, idx).trim();
|
||||||
|
const value = trimmed.slice(idx + 1).trim();
|
||||||
|
if (!process.env[key]) process.env[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseMysqlConfig() {
|
||||||
|
if (process.env.DATABASE_URL) {
|
||||||
|
const url = new URL(process.env.DATABASE_URL);
|
||||||
|
if (url.protocol !== 'mysql:') {
|
||||||
|
throw new Error(`Unsupported DATABASE_URL scheme for SLO report: ${url.protocol}`);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
host: url.hostname,
|
||||||
|
port: Number(url.port || 3306),
|
||||||
|
user: decodeURIComponent(url.username),
|
||||||
|
password: decodeURIComponent(url.password),
|
||||||
|
database: url.pathname.replace(/^\/+/, ''),
|
||||||
|
charset: 'utf8mb4',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
host: process.env.MYSQL_HOST,
|
||||||
|
port: Number(process.env.MYSQL_PORT || 3306),
|
||||||
|
user: process.env.MYSQL_USER,
|
||||||
|
password: process.env.MYSQL_PASSWORD,
|
||||||
|
database: process.env.MYSQL_DATABASE,
|
||||||
|
charset: 'utf8mb4',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function walkCount(root) {
|
||||||
|
const stack = [root];
|
||||||
|
let files = 0;
|
||||||
|
let dirs = 0;
|
||||||
|
let bytes = 0;
|
||||||
|
while (stack.length) {
|
||||||
|
const current = stack.pop();
|
||||||
|
let entries = [];
|
||||||
|
try {
|
||||||
|
entries = fs.readdirSync(current, { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (const entry of entries) {
|
||||||
|
const full = path.join(current, entry.name);
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
dirs += 1;
|
||||||
|
stack.push(full);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!entry.isFile()) continue;
|
||||||
|
files += 1;
|
||||||
|
try {
|
||||||
|
bytes += fs.statSync(full).size;
|
||||||
|
} catch {
|
||||||
|
// ignore files that disappear while counting
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { files, dirs, bytes };
|
||||||
|
}
|
||||||
|
|
||||||
|
function uniquePaths(paths) {
|
||||||
|
const seen = new Set();
|
||||||
|
const result = [];
|
||||||
|
for (const item of paths) {
|
||||||
|
const resolved = path.resolve(item);
|
||||||
|
if (seen.has(resolved)) continue;
|
||||||
|
seen.add(resolved);
|
||||||
|
result.push(resolved);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tableCountQueries() {
|
||||||
|
return [
|
||||||
|
['h5_users', 'users'],
|
||||||
|
['h5_user_sessions', 'sessions'],
|
||||||
|
['h5_agent_runs', 'agentRuns'],
|
||||||
|
['h5_agent_run_events', 'agentRunEvents'],
|
||||||
|
['h5_capability_grants', 'capabilityGrants'],
|
||||||
|
['h5_llm_provider_keys', 'llmProviderKeys'],
|
||||||
|
['mindspace_assets', 'mindspaceAssets'],
|
||||||
|
['mindspace_pages', 'mindspacePages'],
|
||||||
|
['mindspace_publications', 'mindspacePublications'],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async function safeTableCount(conn, table) {
|
||||||
|
try {
|
||||||
|
const [rows] = await conn.query(`SELECT COUNT(*) AS count FROM \`${table}\``);
|
||||||
|
return Number(rows[0]?.count ?? 0);
|
||||||
|
} catch (err) {
|
||||||
|
return { error: err instanceof Error ? err.message : String(err) };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readDbSummary() {
|
||||||
|
const conn = await mysql.createConnection(parseMysqlConfig());
|
||||||
|
try {
|
||||||
|
const result = {};
|
||||||
|
for (const [table, key] of tableCountQueries()) {
|
||||||
|
result[key] = await safeTableCount(conn, table);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
} finally {
|
||||||
|
await conn.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readRedisSummary(namespace, redisUrl) {
|
||||||
|
const client = createClient({ url: redisUrl });
|
||||||
|
await client.connect();
|
||||||
|
try {
|
||||||
|
const workerKeys = await client.keys(`${namespace}:worker:*:active_streams`);
|
||||||
|
const streamKeys = await client.keys(`${namespace}:stream:*:status`);
|
||||||
|
const sessionKeys = await client.keys(`${namespace}:session:*:target`);
|
||||||
|
return {
|
||||||
|
workerCount: workerKeys.length,
|
||||||
|
streamStatusCount: streamKeys.length,
|
||||||
|
sessionTargetCount: sessionKeys.length,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
await client.quit();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function summarizeRuntime(runtimeJson) {
|
||||||
|
const workers = runtimeJson?.router?.workers ?? [];
|
||||||
|
const staleMetricMs = 2 * 60 * 1000;
|
||||||
|
const now = Date.now();
|
||||||
|
return {
|
||||||
|
routerEnabled: Boolean(runtimeJson?.router?.enabled),
|
||||||
|
publicBaseUrl: runtimeJson?.publicBaseUrl ?? null,
|
||||||
|
toolRuntime: runtimeJson?.toolRuntime ?? null,
|
||||||
|
workers: workers.map((worker) => ({
|
||||||
|
id: worker.id,
|
||||||
|
healthy: runtimeJson?.targets?.find((target) => target.target === worker.target)?.healthy ?? null,
|
||||||
|
activeStreams: worker.activeStreams,
|
||||||
|
streamOpenCount: worker.streamOpenCount,
|
||||||
|
streamAbortCount: worker.streamAbortCount,
|
||||||
|
streamErrorCount: worker.streamErrorCount,
|
||||||
|
cpuLoad: worker.cpuLoad,
|
||||||
|
memoryPressure: worker.memoryPressure,
|
||||||
|
fdPressure: worker.fdPressure,
|
||||||
|
fdCount: worker.fdCount,
|
||||||
|
containerHealth: worker.containerHealth,
|
||||||
|
metricsAgeMs: worker.metricsSampledAt ? now - worker.metricsSampledAt : null,
|
||||||
|
metricsFresh: worker.metricsSampledAt ? now - worker.metricsSampledAt <= staleMetricMs : false,
|
||||||
|
score: worker.score,
|
||||||
|
drain: worker.drain,
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const envFile = process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env');
|
||||||
|
const appRoot = process.env.MEMIND_APP_ROOT || path.dirname(path.resolve(envFile));
|
||||||
|
loadEnvFile(envFile);
|
||||||
|
|
||||||
|
const publicBase = (process.env.H5_PUBLIC_BASE_URL || 'https://mm.tkmind.cn').replace(/\/$/, '');
|
||||||
|
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||||
|
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||||
|
const mindSpaceRoots = uniquePaths([
|
||||||
|
path.join(appRoot, 'MindSpace'),
|
||||||
|
process.env.MINDSPACE_STORAGE_ROOT || path.join(appRoot, 'data', 'mindspace'),
|
||||||
|
]);
|
||||||
|
const dispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
||||||
|
|
||||||
|
async function fetchRuntime() {
|
||||||
|
const res = await undiciFetch(`${publicBase}/api/runtime/status`, {
|
||||||
|
dispatcher: publicBase.startsWith('https://127.0.0.1') ? dispatcher : undefined,
|
||||||
|
});
|
||||||
|
const text = await res.text();
|
||||||
|
return {
|
||||||
|
ok: res.ok,
|
||||||
|
status: res.status,
|
||||||
|
json: JSON.parse(text),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const runtime = await fetchRuntime().catch((err) => ({ ok: false, error: err.message }));
|
||||||
|
const runtimeSummary = runtime.ok ? summarizeRuntime(runtime.json) : null;
|
||||||
|
const db = await readDbSummary().catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
|
||||||
|
const redis = await readRedisSummary(namespace, redisUrl).catch((err) => ({
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
}));
|
||||||
|
const mindSpace = mindSpaceRoots.map((root) => (
|
||||||
|
fs.existsSync(root)
|
||||||
|
? { root, ...walkCount(root) }
|
||||||
|
: { root, error: 'missing' }
|
||||||
|
));
|
||||||
|
|
||||||
|
const failures = [];
|
||||||
|
if (!runtime.ok) failures.push('runtime_status_unavailable');
|
||||||
|
if (runtimeSummary && !runtimeSummary.routerEnabled) failures.push('router_disabled');
|
||||||
|
for (const worker of runtimeSummary?.workers ?? []) {
|
||||||
|
if (!worker.healthy) failures.push(`${worker.id}_target_unhealthy`);
|
||||||
|
if (worker.containerHealth && worker.containerHealth !== 'healthy') failures.push(`${worker.id}_container_${worker.containerHealth}`);
|
||||||
|
if (!worker.metricsFresh) failures.push(`${worker.id}_metrics_stale`);
|
||||||
|
if (worker.streamErrorCount > 0) failures.push(`${worker.id}_stream_errors_${worker.streamErrorCount}`);
|
||||||
|
}
|
||||||
|
if (runtimeSummary?.toolRuntime?.chatInjectsCodeTools !== false) failures.push('chat_injects_code_tools');
|
||||||
|
|
||||||
|
const report = {
|
||||||
|
ok: failures.length === 0,
|
||||||
|
checkedAt: new Date().toISOString(),
|
||||||
|
publicBase,
|
||||||
|
appRoot,
|
||||||
|
envFile,
|
||||||
|
namespace,
|
||||||
|
runtime: runtimeSummary,
|
||||||
|
db,
|
||||||
|
redis,
|
||||||
|
mindSpace,
|
||||||
|
failures,
|
||||||
|
writes: {
|
||||||
|
database: false,
|
||||||
|
mindSpace: false,
|
||||||
|
redis: false,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
console.log(JSON.stringify(report, null, 2));
|
||||||
|
process.exit(report.ok ? 0 : 1);
|
||||||
Reference in New Issue
Block a user