fix: let external agent worker drain before exit

This commit is contained in:
John
2026-07-02 08:30:01 +08:00
parent a402509d6e
commit bb98734d9c
4 changed files with 42 additions and 5 deletions
+20 -2
View File
@@ -286,7 +286,7 @@ function createAgentRunGateway({
error_message: message, error_message: message,
completed_at: retryable ? null : nowMs() completed_at: retryable ? null : nowMs()
}); });
if (retryable) { if (retryable && autoDispatch) {
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]); setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
} }
} }
@@ -6888,13 +6888,27 @@ process.on("SIGTERM", () => {
}); });
async function runOnce() { async function runOnce() {
const result = await gateway.dispatchQueuedRuns({ limit: args.limit }); const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
await waitForIdle();
console.log(JSON.stringify({ console.log(JSON.stringify({
ok: true, ok: true,
mode: args.status ? "status" : "dispatch", mode: args.status ? "status" : "dispatch",
dispatched: args.status ? 0 : result.dispatched, dispatched: args.status ? 0 : result.dispatched,
queue: result.queue queue: await gateway.getQueueStatus()
})); }));
} }
async function waitForIdle() {
const startedAt = Date.now();
const maxWaitMs = Number(process.env.MEMIND_AGENT_RUN_WORKER_DRAIN_WAIT_MS ?? 20 * 60 * 1e3);
while (!stopping) {
const status = await gateway.getQueueStatus();
if (status.inFlight === 0 && status.pendingDispatches === 0) return;
if (Date.now() - startedAt > maxWaitMs) {
throw new Error(`agent run worker did not drain within ${maxWaitMs}ms`);
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
}
var exitCode = 0;
try { try {
if (args.status) { if (args.status) {
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2)); console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
@@ -6907,7 +6921,11 @@ try {
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs))); await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
} }
} }
} catch (err) {
exitCode = 1;
console.error(err instanceof Error ? err.message : String(err));
} finally { } finally {
await pool.end().catch(() => { await pool.end().catch(() => {
}); });
if (args.status || args.once) process.exit(exitCode);
} }
+1 -1
View File
@@ -6783,7 +6783,7 @@ function createAgentRunGateway({
error_message: message, error_message: message,
completed_at: retryable ? null : nowMs() completed_at: retryable ? null : nowMs()
}); });
if (retryable) { if (retryable && autoDispatch) {
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]); setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
} }
} }
+1 -1
View File
@@ -310,7 +310,7 @@ export function createAgentRunGateway({
error_message: message, error_message: message,
completed_at: retryable ? null : nowMs(), completed_at: retryable ? null : nowMs(),
}); });
if (retryable) { if (retryable && autoDispatch) {
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]); setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
} }
} }
+20 -1
View File
@@ -96,14 +96,29 @@ process.on('SIGTERM', () => { stopping = true; });
async function runOnce() { async function runOnce() {
const result = await gateway.dispatchQueuedRuns({ limit: args.limit }); const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
await waitForIdle();
console.log(JSON.stringify({ console.log(JSON.stringify({
ok: true, ok: true,
mode: args.status ? 'status' : 'dispatch', mode: args.status ? 'status' : 'dispatch',
dispatched: args.status ? 0 : result.dispatched, dispatched: args.status ? 0 : result.dispatched,
queue: result.queue, queue: await gateway.getQueueStatus(),
})); }));
} }
async function waitForIdle() {
const startedAt = Date.now();
const maxWaitMs = Number(process.env.MEMIND_AGENT_RUN_WORKER_DRAIN_WAIT_MS ?? 20 * 60 * 1000);
while (!stopping) {
const status = await gateway.getQueueStatus();
if (status.inFlight === 0 && status.pendingDispatches === 0) return;
if (Date.now() - startedAt > maxWaitMs) {
throw new Error(`agent run worker did not drain within ${maxWaitMs}ms`);
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
}
let exitCode = 0;
try { try {
if (args.status) { if (args.status) {
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2)); console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
@@ -116,6 +131,10 @@ try {
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs))); await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
} }
} }
} catch (err) {
exitCode = 1;
console.error(err instanceof Error ? err.message : String(err));
} finally { } finally {
await pool.end().catch(() => {}); await pool.end().catch(() => {});
if (args.status || args.once) process.exit(exitCode);
} }