tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: harden orchestrator execution runtime

Browse Source
This commit is contained in:
john
2026-07-24 23:53:32 +08:00
parent 396bb78200
commit f6f2cd0933
55 changed files with 5122 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent-run-gateway.mjs
+123 -7
View File
@@ -99,6 +99,24 @@ function positiveInteger(value, fallback) {
return Math.floor(n);
}
export function normalizeAgentRunWorkerIdentity(identity = {}) {
const runtimeRoot = path.resolve(
String(identity.runtimeRoot ?? process.cwd()).trim() || process.cwd(),
);
const buildId =
String(
identity.buildId
?? process.env.MEMIND_RUNTIME_BUILD_ID
?? process.env.MEMIND_RELEASE_ID
?? process.env.GIT_COMMIT
?? 'dev',
).trim() || 'dev';
const workerId =
String(identity.workerId ?? '').trim()
|| `${process.pid}:${crypto.randomUUID()}`;
return Object.freeze({ workerId, runtimeRoot, buildId });
}
function timeoutError(ms) {
const err = new Error(`agent run timed out after ${ms}ms`);
err.code = 'AGENT_RUN_TIMEOUT';
@@ -292,6 +310,7 @@ export function createAgentRunGateway({
observeWorkflowRun = null,
isSessionExternallyBusy = null,
validateRunDeliverables = null,
quiesceSessionOnTerminal = null,
retryDelaysMs = DEFAULT_RUN_RETRY_DELAYS_MS,
autoDispatch = envFlag(process.env.MEMIND_AGENT_RUN_AUTODISPATCH, true),
maxConcurrentRuns = positiveInteger(
@@ -306,7 +325,9 @@ export function createAgentRunGateway({
process.env.MEMIND_AGENT_RUN_HEARTBEAT_MS,
DEFAULT_RUN_HEARTBEAT_MS,
),
workerIdentity = null,
}) {
const worker = normalizeAgentRunWorkerIdentity(workerIdentity ?? {});
const sessionStore = resolveSessionAccess({ userAuth, sessionAccess });
const inFlight = new Set();
const queuedDispatches = [];
@@ -339,6 +360,41 @@ export function createAgentRunGateway({
);
}
async function quiesceTerminalSession(runId, {
userId,
sessionId,
requestId = null,
status,
}) {
if (!sessionId || typeof quiesceSessionOnTerminal !== 'function') return;
try {
const result = await quiesceSessionOnTerminal({
runId,
userId,
sessionId,
requestId,
status,
});
await appendEvent(runId, 'session_extensions_quiesced', {
sessionId,
status,
removed: result?.removed ?? [],
skipped: Boolean(result?.skipped),
cancellation: result?.cancellation ?? null,
});
} catch (err) {
await appendEvent(runId, 'session_extension_quiesce_failed', {
sessionId,
status,
error: err instanceof Error ? err.message : String(err),
}).catch(() => {});
console.warn(
`[AgentRun] failed to quiesce session ${sessionId}:`,
err instanceof Error ? err.message : err,
);
}
}
async function appendExecutionPlanEvent(input, source) {
const executionPlan = source?.executionPlan;
if (!executionPlan?.dryRun) return;
@@ -490,8 +546,11 @@ export function createAgentRunGateway({
await pool.query(
`INSERT INTO h5_agent_runs
(id, user_id, agent_session_id, request_id, status, attempts,
user_message_json, error_message, created_at, updated_at, started_at, completed_at)
VALUES (?, ?, ?, ?, 'queued', 0, ?, NULL, ?, ?, NULL, NULL)`,
user_message_json, error_message, created_at, updated_at, started_at, completed_at,
required_runtime_root, required_build_id,
claimed_worker_id, claimed_runtime_root, claimed_build_id)
VALUES (?, ?, ?, ?, 'queued', 0, ?, NULL, ?, ?, NULL, NULL,
?, ?, NULL, NULL, NULL)`,
[
runId,
userId,
@@ -500,6 +559,8 @@ export function createAgentRunGateway({
serializeMessage(runMessage),
createdAt,
createdAt,
worker.runtimeRoot,
worker.buildId,
],
);
await appendEvent(runId, 'queued', {
@@ -507,6 +568,8 @@ export function createAgentRunGateway({
toolMode: normalizedToolMode,
taskType: normalizedTaskType,
forceDeepReasoning: Boolean(forceDeepReasoning),
requiredRuntimeRoot: worker.runtimeRoot,
requiredBuildId: worker.buildId,
});
const createdRun = projectRun(await getRunById(runId));
dispatchShadowObservation({
@@ -596,6 +659,9 @@ export function createAgentRunGateway({
attempt,
pid: process.pid,
heartbeatMs,
workerId: worker.workerId,
runtimeRoot: worker.runtimeRoot,
buildId: worker.buildId,
});
} catch (err) {
console.error('[AgentRun] worker heartbeat failed:', err instanceof Error ? err.message : err);
@@ -1112,12 +1178,32 @@ export function createAgentRunGateway({
const nextAttempt = Number(row.attempts ?? 0) + 1;
const [claim] = await pool.query(
`UPDATE h5_agent_runs
SET status = 'running', attempts = ?, started_at = COALESCE(started_at, ?), updated_at = ?, error_message = NULL
WHERE id = ? AND status IN ('queued', 'retryable')`,
[nextAttempt, nowMs(), nowMs(), runId],
SET status = 'running', attempts = ?, started_at = COALESCE(started_at, ?),
updated_at = ?, error_message = NULL,
claimed_worker_id = ?, claimed_runtime_root = ?, claimed_build_id = ?
WHERE id = ?
AND status IN ('queued', 'retryable')
AND (required_runtime_root IS NULL OR required_runtime_root = ?)
AND (required_build_id IS NULL OR required_build_id = ?)`,
[
nextAttempt,
nowMs(),
nowMs(),
worker.workerId,
worker.runtimeRoot,
worker.buildId,
runId,
worker.runtimeRoot,
worker.buildId,
],
);
if (Number(claim?.affectedRows ?? 0) === 0) return;
await appendEvent(runId, 'running', { attempt: nextAttempt });
await appendEvent(runId, 'running', {
attempt: nextAttempt,
workerId: worker.workerId,
runtimeRoot: worker.runtimeRoot,
buildId: worker.buildId,
});
const stopHeartbeat = startRunHeartbeat(runId, { attempt: nextAttempt });
try {
@@ -1145,12 +1231,28 @@ export function createAgentRunGateway({
await markRun(runId, retryable ? 'retryable' : 'failed', {
error_message: message,
completed_at: retryable ? null : nowMs(),
...(retryable
? {
claimed_worker_id: null,
claimed_runtime_root: null,
claimed_build_id: null,
}
: {}),
}, { expectedStatus: 'running' });
if (retryable && autoDispatch) {
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
}
} finally {
stopHeartbeat();
const terminalRun = await getRunById(runId).catch(() => null);
if (terminalRun && TERMINAL_STATUSES.has(terminalRun.status)) {
await quiesceTerminalSession(runId, {
userId: terminalRun.user_id ?? row.user_id,
sessionId: terminalRun.agent_session_id ?? row.agent_session_id ?? null,
requestId: terminalRun.request_id ?? row.request_id ?? null,
status: terminalRun.status,
});
}
}
}
@@ -1354,6 +1456,12 @@ export function createAgentRunGateway({
if (!marked) continue;
item.status = 'succeeded';
item.recoveredAs = 'deliverables';
await quiesceTerminalSession(row.id, {
userId: row.user_id,
sessionId: row.agent_session_id ?? null,
requestId: row.request_id ?? null,
status: 'succeeded',
});
recovered.push(item);
continue;
}
@@ -1401,6 +1509,12 @@ export function createAgentRunGateway({
error: message,
});
item.status = 'failed';
await quiesceTerminalSession(row.id, {
userId: row.user_id,
sessionId: row.agent_session_id ?? null,
requestId: row.request_id ?? null,
status: 'failed',
});
}
recovered.push(item);
}
@@ -1437,9 +1551,11 @@ export function createAgentRunGateway({
`SELECT id
FROM h5_agent_runs
WHERE status IN ('queued', 'retryable')
AND (required_runtime_root IS NULL OR required_runtime_root = ?)
AND (required_build_id IS NULL OR required_build_id = ?)
ORDER BY updated_at ASC
LIMIT ?`,
[take],
[worker.runtimeRoot, worker.buildId, take],
);
for (const row of rows) {
dispatchRun(row.id);