docs(ops): record 103 goosed native migration and add migrate tooling
Memind CI / Test, build, and release guards (pull_request) Has been cancelled
Memind CI / Test, build, and release guards (pull_request) Has been cancelled
Document the completed Docker-to-native goosed pool cutover on 103, add the production migrate script with backup/rollback paths, and extend local native pool soak/metrics helpers used as migration gates. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -54,16 +54,40 @@ function parseMemUsage(value) {
|
||||
};
|
||||
}
|
||||
|
||||
function resolveRuntimeMode() {
|
||||
const configured = String(process.env.GOOSED_RUNTIME ?? 'auto').trim().toLowerCase();
|
||||
if (configured === 'native' || configured === 'docker') return configured;
|
||||
try {
|
||||
const line = sh("docker ps --filter 'label=com.docker.compose.project=goosed-prod' --format '{{.Names}}' 2>/dev/null | head -1");
|
||||
return line ? 'docker' : 'native';
|
||||
} catch {
|
||||
return 'native';
|
||||
}
|
||||
}
|
||||
|
||||
function portFromTarget(target) {
|
||||
try {
|
||||
return new URL(target).port || '443';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function targetWorkerIds() {
|
||||
const targets = (process.env.TKMIND_API_TARGETS || process.env.TKMIND_API_TARGET || '')
|
||||
.split(',')
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
return targets.map((target, index) => ({
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
}));
|
||||
return targets.map((target, index) => {
|
||||
const port = portFromTarget(target);
|
||||
return {
|
||||
id: `goosed-${index + 1}`,
|
||||
target,
|
||||
port,
|
||||
container: `goosed-prod-${index + 1}`,
|
||||
launchdLabel: port ? `com.tkmind.local-goosed-${port}` : null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function readDockerStats(container) {
|
||||
@@ -92,26 +116,51 @@ function readContainerFdCount(container) {
|
||||
}
|
||||
}
|
||||
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
function readListenPid(port) {
|
||||
if (!port) return null;
|
||||
try {
|
||||
const pid = sh(`lsof -nP -iTCP:${port} -sTCP:LISTEN -t 2>/dev/null | head -1`);
|
||||
const n = Number(pid);
|
||||
return Number.isFinite(n) && n > 0 ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
process.exit(2);
|
||||
function readNativeFdCount(pid) {
|
||||
if (!pid) return 0;
|
||||
try {
|
||||
return Number(sh(`lsof -n -p ${pid} 2>/dev/null | wc -l | tr -d ' '`)) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
const workers = [];
|
||||
for (const worker of targetWorkerIds()) {
|
||||
function readNativeProcessStats(pid) {
|
||||
if (!pid) return null;
|
||||
try {
|
||||
const line = sh(`ps -p ${pid} -o %cpu=,rss= 2>/dev/null | tail -1`);
|
||||
const match = line.trim().match(/^([\d.]+)\s+(\d+)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
cpuLoad: Number(match[1]) || 0,
|
||||
rssBytes: (Number(match[2]) || 0) * 1024,
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readLaunchdState(label) {
|
||||
if (!label) return null;
|
||||
try {
|
||||
return sh(`launchctl print gui/$(id -u)/${label} 2>/dev/null | awk '/state =/ {print $3; exit}'`) || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sampleDockerWorker(worker, fdWarn, now) {
|
||||
const stats = readDockerStats(worker.container);
|
||||
const state = readDockerInspect(worker.container);
|
||||
const fdCount = readContainerFdCount(worker.container);
|
||||
@@ -119,8 +168,9 @@ for (const worker of targetWorkerIds()) {
|
||||
const cpuLoad = parsePercent(stats?.CPUPerc);
|
||||
const memoryPressure = parsePercent(stats?.MemPerc);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
workers.push({
|
||||
return {
|
||||
...worker,
|
||||
runtime: 'docker',
|
||||
ok: Boolean(stats && state?.Running),
|
||||
health: state?.Health?.Status ?? state?.Status ?? null,
|
||||
hostPid: Number(state?.Pid ?? 0) || null,
|
||||
@@ -132,9 +182,60 @@ for (const worker of targetWorkerIds()) {
|
||||
memUsedBytes: mem.usedBytes,
|
||||
memLimitBytes: mem.limitBytes,
|
||||
sampledAt: now,
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function sampleNativeWorker(worker, fdWarn, now) {
|
||||
const hostPid = readListenPid(worker.port);
|
||||
const fdCount = readNativeFdCount(hostPid);
|
||||
const proc = readNativeProcessStats(hostPid);
|
||||
const launchdState = readLaunchdState(worker.launchdLabel);
|
||||
const fdPressure = fdWarn > 0 ? Number((fdCount / fdWarn).toFixed(4)) : 0;
|
||||
const memUsedBytes = proc?.rssBytes ?? 0;
|
||||
return {
|
||||
...worker,
|
||||
runtime: 'native',
|
||||
ok: Boolean(hostPid && launchdState === 'running'),
|
||||
health: launchdState,
|
||||
hostPid,
|
||||
cpuLoad: proc?.cpuLoad ?? 0,
|
||||
memoryPressure: 0,
|
||||
fdPressure,
|
||||
fdCount,
|
||||
pids: hostPid ? 1 : 0,
|
||||
memUsedBytes,
|
||||
memLimitBytes: 0,
|
||||
sampledAt: now,
|
||||
container: worker.launchdLabel ?? worker.container,
|
||||
};
|
||||
}
|
||||
|
||||
function workerKey(namespace, id, field) {
|
||||
return [namespace, 'worker', id, field].join(':');
|
||||
}
|
||||
|
||||
loadEnvFile(process.env.MEMIND_ENV_FILE || path.join(process.cwd(), '.env'));
|
||||
|
||||
const runtimeMode = resolveRuntimeMode();
|
||||
const redisUrl = process.env.MEMIND_RUNTIME_REDIS_URL || 'redis://127.0.0.1:6379/0';
|
||||
const namespace = process.env.MEMIND_RUNTIME_REDIS_NAMESPACE || 'memind:runtime';
|
||||
const action = process.argv[2] || 'sample';
|
||||
const dryRun = process.argv.includes('--dry-run') || action === 'status';
|
||||
const fdWarn = Number(process.env.GOOSED_FD_WARN ?? 180);
|
||||
const now = Date.now();
|
||||
|
||||
if (!['sample', 'status'].includes(action)) {
|
||||
console.error('Usage: node scripts/runtime-worker-metrics.mjs <sample|status> [--dry-run]');
|
||||
console.error('Env: GOOSED_RUNTIME=auto|native|docker (default auto)');
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const workers = targetWorkerIds().map((worker) => (
|
||||
runtimeMode === 'native'
|
||||
? sampleNativeWorker(worker, fdWarn, now)
|
||||
: sampleDockerWorker(worker, fdWarn, now)
|
||||
));
|
||||
|
||||
if (!dryRun) {
|
||||
const client = createClient({ url: redisUrl });
|
||||
client.on('error', (err) => {
|
||||
@@ -163,6 +264,7 @@ console.log(JSON.stringify({
|
||||
ok: workers.every((worker) => worker.ok),
|
||||
action,
|
||||
dryRun,
|
||||
runtimeMode,
|
||||
namespace,
|
||||
redisWrites: !dryRun,
|
||||
workers,
|
||||
|
||||
Reference in New Issue
Block a user