fix: harden agent and WeChat run completion
This commit is contained in:
+20
-13
@@ -43,12 +43,16 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
|
||||
return timestamps.length ? Math.max(...timestamps) : null;
|
||||
};
|
||||
|
||||
const isStaleRunningRow = (row, startedCutoff, sessionFinishedCutoff) => {
|
||||
const isStaleRunningRow = (row, heartbeatCutoff, startedCutoff, sessionFinishedCutoff) => {
|
||||
if (row.status !== 'running' || row.started_at == null) return false;
|
||||
const finishedAt = sessionFinishedAt(row.id);
|
||||
const heartbeatAt = latestHeartbeatAt(row.id);
|
||||
return (
|
||||
Number(row.started_at) <= Number(startedCutoff)
|
||||
|| (finishedAt != null && Number(finishedAt) <= Number(sessionFinishedCutoff))
|
||||
(heartbeatAt == null || Number(heartbeatAt) <= Number(heartbeatCutoff))
|
||||
&& (
|
||||
Number(row.started_at) <= Number(startedCutoff)
|
||||
|| (finishedAt != null && Number(finishedAt) <= Number(sessionFinishedCutoff))
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
@@ -112,9 +116,9 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
|
||||
}]];
|
||||
}
|
||||
if (sql.includes('SELECT') && sql.includes('session_finished_at') && sql.includes('r.started_at <= ?')) {
|
||||
const [startedCutoff, sessionFinishedCutoff, limit = 1] = params;
|
||||
const [heartbeatCutoff, startedCutoff, sessionFinishedCutoff, limit = 1] = params;
|
||||
return [[...runs.values()]
|
||||
.filter((row) => isStaleRunningRow(row, startedCutoff, sessionFinishedCutoff))
|
||||
.filter((row) => isStaleRunningRow(row, heartbeatCutoff, startedCutoff, sessionFinishedCutoff))
|
||||
.sort((a, b) => {
|
||||
const aKey = Number(sessionFinishedAt(a.id) ?? a.started_at ?? 0);
|
||||
const bKey = Number(sessionFinishedAt(b.id) ?? b.started_at ?? 0);
|
||||
@@ -237,9 +241,9 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
|
||||
return [{ affectedRows: 1 }];
|
||||
}
|
||||
if (sql.includes("WHERE id = ?") && sql.includes("status = 'running'") && sql.includes('session_finished')) {
|
||||
const [errorMessage, updatedAt, completedAt, id, startedCutoff, sessionFinishedCutoff] = params;
|
||||
const [errorMessage, updatedAt, completedAt, id, startedCutoff, sessionFinishedCutoff, heartbeatCutoff] = params;
|
||||
const row = runs.get(id);
|
||||
if (!row || !isStaleRunningRow(row, startedCutoff, sessionFinishedCutoff)) {
|
||||
if (!row || !isStaleRunningRow(row, heartbeatCutoff, startedCutoff, sessionFinishedCutoff)) {
|
||||
return [{ affectedRows: 0 }];
|
||||
}
|
||||
Object.assign(row, {
|
||||
@@ -283,7 +287,9 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
|
||||
return [rows];
|
||||
}
|
||||
if (sql.includes('UPDATE h5_agent_runs SET')) {
|
||||
const id = params.at(-1);
|
||||
const setSql = sql.split(' WHERE ')[0];
|
||||
const columns = [...setSql.matchAll(/([a-z_]+) = \?/g)].map((match) => match[1]);
|
||||
const id = params[columns.length];
|
||||
const row = runs.get(id);
|
||||
if (!row) return [{ affectedRows: 0 }];
|
||||
if (sql.includes("status = 'running'") && sql.includes('started_at = COALESCE(started_at, ?)')) {
|
||||
@@ -297,7 +303,8 @@ function createFakePool({ sessionDeliverables = {}, workspaceDeliverables = {} }
|
||||
});
|
||||
return [{ affectedRows: 1 }];
|
||||
}
|
||||
const columns = [...sql.matchAll(/([a-z_]+) = \?/g)].map((match) => match[1]);
|
||||
const expectedStatus = sql.includes('AND status = ?') ? params[columns.length + 1] : null;
|
||||
if (expectedStatus && row.status !== expectedStatus) return [{ affectedRows: 0 }];
|
||||
for (let i = 0; i < columns.length; i += 1) {
|
||||
row[columns[i]] = params[i];
|
||||
}
|
||||
@@ -1817,7 +1824,7 @@ test('stale running recovery marks old running rows failed with an event', async
|
||||
);
|
||||
});
|
||||
|
||||
test('stale running recovery still considers old runs even with fresh heartbeat', async () => {
|
||||
test('stale running recovery ignores old runs with a fresh heartbeat', async () => {
|
||||
const pool = createFakePool();
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
@@ -1847,9 +1854,9 @@ test('stale running recovery still considers old runs even with fresh heartbeat'
|
||||
|
||||
const result = await gateway.recoverStaleRunningRuns({ staleMs: 1000, dryRun: false });
|
||||
|
||||
assert.equal(result.considered, 1);
|
||||
assert.equal(result.recovered, 1);
|
||||
assert.equal(pool.runs.get(run.id).status, 'failed');
|
||||
assert.equal(result.considered, 0);
|
||||
assert.equal(result.recovered, 0);
|
||||
assert.equal(pool.runs.get(run.id).status, 'running');
|
||||
});
|
||||
|
||||
test('stale running recovery succeeds when workspace pages exist after sync', async () => {
|
||||
|
||||
Reference in New Issue
Block a user