0dfb2e2b82
Memind CI / Test, build, and release guards (pull_request) Has been cancelled
Document the completed Docker-to-native goosed pool cutover on 103, add the production migrate script with backup/rollback paths, and extend local native pool soak/metrics helpers used as migration gates. Co-authored-by: Cursor <cursoragent@cursor.com>
225 lines
6.2 KiB
JavaScript
225 lines
6.2 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Local native goosed pool soak: periodic /status + FD sampling across TKMIND_API_TARGETS.
|
|
*
|
|
* Usage:
|
|
* node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]
|
|
*/
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { execFileSync } from 'node:child_process';
|
|
import { Agent, fetch as undiciFetch } from 'undici';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const root = path.join(__dirname, '..');
|
|
const insecureDispatcher = new Agent({ connect: { rejectUnauthorized: false } });
|
|
|
|
function loadEnvFile(filePath) {
|
|
if (!fs.existsSync(filePath)) return;
|
|
for (const line of fs.readFileSync(filePath, 'utf8').split('\n')) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith('#')) continue;
|
|
const idx = trimmed.indexOf('=');
|
|
if (idx < 0) continue;
|
|
const key = trimmed.slice(0, idx).trim();
|
|
const value = trimmed.slice(idx + 1).trim();
|
|
if (!process.env[key]) process.env[key] = value;
|
|
}
|
|
}
|
|
|
|
function sh(command) {
|
|
return execFileSync('/bin/zsh', ['-lc', command], { encoding: 'utf8' }).trim();
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const options = {
|
|
minutes: Number(process.env.GOOSED_SOAK_MINUTES ?? 30),
|
|
intervalSec: Number(process.env.GOOSED_SOAK_INTERVAL_SEC ?? 60),
|
|
};
|
|
for (let i = 0; i < argv.length; i += 1) {
|
|
const arg = argv[i];
|
|
if (arg === '--minutes') {
|
|
options.minutes = Number(argv[++i]);
|
|
} else if (arg === '--interval') {
|
|
options.intervalSec = Number(argv[++i]);
|
|
} else if (arg === '-h' || arg === '--help') {
|
|
console.log(`Usage: node scripts/soak-local-goosed-pool.mjs [--minutes 30] [--interval 60]`);
|
|
process.exit(0);
|
|
}
|
|
}
|
|
if (!Number.isFinite(options.minutes) || options.minutes <= 0) {
|
|
throw new Error('invalid --minutes');
|
|
}
|
|
if (!Number.isFinite(options.intervalSec) || options.intervalSec <= 0) {
|
|
throw new Error('invalid --interval');
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function resolveTargets() {
|
|
return (process.env.TKMIND_API_TARGETS || process.env.GOOSED_NATIVE_POOL_PORTS || '18006,18007')
|
|
.split(/[,\s]+/)
|
|
.map((value) => value.trim())
|
|
.filter(Boolean)
|
|
.flatMap((value) => (
|
|
value.startsWith('http')
|
|
? [value.replace(/\/$/, '')]
|
|
: [`https://127.0.0.1:${value.replace(/^:/, '')}`]
|
|
));
|
|
}
|
|
|
|
function portFromTarget(target) {
|
|
try {
|
|
return new URL(target).port || '443';
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function nativeLaunchdLabel(port) {
|
|
return port ? `com.tkmind.local-goosed-${port}` : null;
|
|
}
|
|
|
|
function readListenPid(port) {
|
|
if (!port) return null;
|
|
try {
|
|
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
|
const n = Number(pid);
|
|
return Number.isFinite(n) && n > 0 ? n : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function readFdCount(pid) {
|
|
if (!pid) return 0;
|
|
try {
|
|
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
|
} catch {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
function readLaunchdState(label) {
|
|
if (!label) return null;
|
|
try {
|
|
const output = sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`);
|
|
return output || null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function probeTarget(target) {
|
|
const started = Date.now();
|
|
try {
|
|
const res = await undiciFetch(`${target}/status`, { dispatcher: insecureDispatcher });
|
|
const body = (await res.text()).trim();
|
|
return {
|
|
ok: res.ok && body === 'ok',
|
|
status: res.status,
|
|
body,
|
|
latencyMs: Date.now() - started,
|
|
};
|
|
} catch (err) {
|
|
return {
|
|
ok: false,
|
|
latencyMs: Date.now() - started,
|
|
error: err instanceof Error ? err.message : String(err),
|
|
};
|
|
}
|
|
}
|
|
|
|
function sleep(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
loadEnvFile(path.join(root, '.env'));
|
|
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const targets = resolveTargets();
|
|
if (targets.length === 0) {
|
|
console.error('No targets configured.');
|
|
process.exit(2);
|
|
}
|
|
|
|
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
|
const fdRestart = Number(process.env.GOOSED_FD_RESTART ?? 3200);
|
|
const endAt = Date.now() + options.minutes * 60_000;
|
|
const summary = {
|
|
startedAt: new Date().toISOString(),
|
|
minutes: options.minutes,
|
|
intervalSec: options.intervalSec,
|
|
targets,
|
|
ticks: 0,
|
|
failures: 0,
|
|
maxFdByTarget: Object.fromEntries(targets.map((target) => [target, 0])),
|
|
samples: [],
|
|
};
|
|
|
|
console.log(`[soak] starting ${options.minutes}m interval=${options.intervalSec}s targets=${targets.join(', ')}`);
|
|
|
|
while (Date.now() < endAt) {
|
|
summary.ticks += 1;
|
|
const tickStarted = Date.now();
|
|
const tick = {
|
|
at: new Date().toISOString(),
|
|
targets: [],
|
|
ok: true,
|
|
};
|
|
|
|
for (const target of targets) {
|
|
const port = portFromTarget(target);
|
|
const label = nativeLaunchdLabel(port);
|
|
const pid = readListenPid(port);
|
|
const fdCount = readFdCount(pid);
|
|
const launchdState = readLaunchdState(label);
|
|
const probe = await probeTarget(target);
|
|
|
|
summary.maxFdByTarget[target] = Math.max(summary.maxFdByTarget[target], fdCount);
|
|
const entry = {
|
|
target,
|
|
port,
|
|
launchdLabel: label,
|
|
launchdState,
|
|
pid,
|
|
fdCount,
|
|
fdWarn,
|
|
fdRestart,
|
|
fdPressure: fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0,
|
|
...probe,
|
|
};
|
|
tick.targets.push(entry);
|
|
if (!entry.ok || fdCount >= fdWarn) tick.ok = false;
|
|
|
|
const status = entry.ok ? 'ok' : 'FAIL';
|
|
console.log(
|
|
`[soak] tick=${summary.ticks} ${target} ${status} pid=${pid ?? '-'} fd=${fdCount} launchd=${launchdState ?? '-'} latency=${entry.latencyMs}ms`,
|
|
);
|
|
}
|
|
|
|
if (!tick.ok) summary.failures += 1;
|
|
summary.samples.push(tick);
|
|
|
|
const elapsed = Date.now() - tickStarted;
|
|
const waitMs = Math.max(0, options.intervalSec * 1000 - elapsed);
|
|
if (Date.now() + waitMs >= endAt) break;
|
|
await sleep(waitMs);
|
|
}
|
|
|
|
summary.finishedAt = new Date().toISOString();
|
|
summary.ok = summary.failures === 0;
|
|
|
|
console.log(JSON.stringify({
|
|
ok: summary.ok,
|
|
ticks: summary.ticks,
|
|
failures: summary.failures,
|
|
maxFdByTarget: summary.maxFdByTarget,
|
|
fdWarn,
|
|
fdRestart,
|
|
}, null, 2));
|
|
|
|
insecureDispatcher.close();
|
|
process.exit(summary.ok ? 0 : 1);
|