fix: let external agent worker drain before exit
This commit is contained in:
@@ -286,7 +286,7 @@ function createAgentRunGateway({
|
|||||||
error_message: message,
|
error_message: message,
|
||||||
completed_at: retryable ? null : nowMs()
|
completed_at: retryable ? null : nowMs()
|
||||||
});
|
});
|
||||||
if (retryable) {
|
if (retryable && autoDispatch) {
|
||||||
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -6888,13 +6888,27 @@ process.on("SIGTERM", () => {
|
|||||||
});
|
});
|
||||||
async function runOnce() {
|
async function runOnce() {
|
||||||
const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
|
const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
|
||||||
|
await waitForIdle();
|
||||||
console.log(JSON.stringify({
|
console.log(JSON.stringify({
|
||||||
ok: true,
|
ok: true,
|
||||||
mode: args.status ? "status" : "dispatch",
|
mode: args.status ? "status" : "dispatch",
|
||||||
dispatched: args.status ? 0 : result.dispatched,
|
dispatched: args.status ? 0 : result.dispatched,
|
||||||
queue: result.queue
|
queue: await gateway.getQueueStatus()
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
async function waitForIdle() {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const maxWaitMs = Number(process.env.MEMIND_AGENT_RUN_WORKER_DRAIN_WAIT_MS ?? 20 * 60 * 1e3);
|
||||||
|
while (!stopping) {
|
||||||
|
const status = await gateway.getQueueStatus();
|
||||||
|
if (status.inFlight === 0 && status.pendingDispatches === 0) return;
|
||||||
|
if (Date.now() - startedAt > maxWaitMs) {
|
||||||
|
throw new Error(`agent run worker did not drain within ${maxWaitMs}ms`);
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var exitCode = 0;
|
||||||
try {
|
try {
|
||||||
if (args.status) {
|
if (args.status) {
|
||||||
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
|
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
|
||||||
@@ -6907,7 +6921,11 @@ try {
|
|||||||
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
|
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (err) {
|
||||||
|
exitCode = 1;
|
||||||
|
console.error(err instanceof Error ? err.message : String(err));
|
||||||
} finally {
|
} finally {
|
||||||
await pool.end().catch(() => {
|
await pool.end().catch(() => {
|
||||||
});
|
});
|
||||||
|
if (args.status || args.once) process.exit(exitCode);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6783,7 +6783,7 @@ function createAgentRunGateway({
|
|||||||
error_message: message,
|
error_message: message,
|
||||||
completed_at: retryable ? null : nowMs()
|
completed_at: retryable ? null : nowMs()
|
||||||
});
|
});
|
||||||
if (retryable) {
|
if (retryable && autoDispatch) {
|
||||||
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -310,7 +310,7 @@ export function createAgentRunGateway({
|
|||||||
error_message: message,
|
error_message: message,
|
||||||
completed_at: retryable ? null : nowMs(),
|
completed_at: retryable ? null : nowMs(),
|
||||||
});
|
});
|
||||||
if (retryable) {
|
if (retryable && autoDispatch) {
|
||||||
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -96,14 +96,29 @@ process.on('SIGTERM', () => { stopping = true; });
|
|||||||
|
|
||||||
async function runOnce() {
|
async function runOnce() {
|
||||||
const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
|
const result = await gateway.dispatchQueuedRuns({ limit: args.limit });
|
||||||
|
await waitForIdle();
|
||||||
console.log(JSON.stringify({
|
console.log(JSON.stringify({
|
||||||
ok: true,
|
ok: true,
|
||||||
mode: args.status ? 'status' : 'dispatch',
|
mode: args.status ? 'status' : 'dispatch',
|
||||||
dispatched: args.status ? 0 : result.dispatched,
|
dispatched: args.status ? 0 : result.dispatched,
|
||||||
queue: result.queue,
|
queue: await gateway.getQueueStatus(),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function waitForIdle() {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const maxWaitMs = Number(process.env.MEMIND_AGENT_RUN_WORKER_DRAIN_WAIT_MS ?? 20 * 60 * 1000);
|
||||||
|
while (!stopping) {
|
||||||
|
const status = await gateway.getQueueStatus();
|
||||||
|
if (status.inFlight === 0 && status.pendingDispatches === 0) return;
|
||||||
|
if (Date.now() - startedAt > maxWaitMs) {
|
||||||
|
throw new Error(`agent run worker did not drain within ${maxWaitMs}ms`);
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let exitCode = 0;
|
||||||
try {
|
try {
|
||||||
if (args.status) {
|
if (args.status) {
|
||||||
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
|
console.log(JSON.stringify({ ok: true, queue: await gateway.getQueueStatus() }, null, 2));
|
||||||
@@ -116,6 +131,10 @@ try {
|
|||||||
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
|
await new Promise((resolve) => setTimeout(resolve, Math.max(200, args.pollMs)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch (err) {
|
||||||
|
exitCode = 1;
|
||||||
|
console.error(err instanceof Error ? err.message : String(err));
|
||||||
} finally {
|
} finally {
|
||||||
await pool.end().catch(() => {});
|
await pool.end().catch(() => {});
|
||||||
|
if (args.status || args.once) process.exit(exitCode);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user