feat(agent-run): retry goose page timeouts with Cursor executor
Memind CI / Test, build, and release guards (push) Failing after 11m45s
Memind CI / Test, build, and release guards (push) Failing after 11m45s
When a page-generation run times out without recoverable HTML, retry once via Cursor code executor if MEMIND_CURSOR_PAGE_TIMEOUT_TAKEOVER is enabled. Also release delivery contracts after successful deliverable recovery. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -44,6 +44,7 @@ import {
|
||||
sanitizeUserFacingBrandText,
|
||||
} from './executor-display-label.mjs';
|
||||
import { applyCursorFirstAgentExecution } from './cursor-page-routing.mjs';
|
||||
import { resolveCursorPageTimeoutTakeover } from './cursor-page-timeout-takeover.mjs';
|
||||
import { cursorDeepseekFallbackEnabled } from './cursor-agent-launch.mjs';
|
||||
import { buildCursorBillingTokenState } from './cursor-agent-usage.mjs';
|
||||
import {
|
||||
@@ -855,6 +856,7 @@ export function createAgentRunGateway({
|
||||
sessionSnapshotService = null,
|
||||
conversationMemoryService = null,
|
||||
syncUserPagesOnSuccess = null,
|
||||
releaseUserPageDeliveryContractsOnSuccess = null,
|
||||
observePersonalMemoryOnSuccess = null,
|
||||
experienceService = null,
|
||||
observeWorkflowRun = null,
|
||||
@@ -1472,6 +1474,46 @@ export function createAgentRunGateway({
|
||||
});
|
||||
}
|
||||
|
||||
function collectDeliverableRelativePaths(deliverables, deliveryResult) {
|
||||
const paths = new Set();
|
||||
for (const page of deliverables?.pages ?? []) {
|
||||
const relativePath = String(page?.workspaceRelativePath ?? '').trim();
|
||||
if (relativePath) paths.add(relativePath);
|
||||
}
|
||||
for (const relativePath of deliveryResult?.publicHtmlRelativePaths ?? []) {
|
||||
const normalized = String(relativePath ?? '').trim();
|
||||
if (normalized) paths.add(normalized);
|
||||
}
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
async function releaseDeliverableContracts({
|
||||
userId,
|
||||
deliverables,
|
||||
deliveryResult,
|
||||
}) {
|
||||
if (typeof releaseUserPageDeliveryContractsOnSuccess !== 'function') {
|
||||
return [];
|
||||
}
|
||||
const relativePaths = collectDeliverableRelativePaths(
|
||||
deliverables,
|
||||
deliveryResult,
|
||||
);
|
||||
if (relativePaths.length === 0) return [];
|
||||
try {
|
||||
return await releaseUserPageDeliveryContractsOnSuccess({
|
||||
userId,
|
||||
relativePaths,
|
||||
});
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[AgentRun] delivery contract release failed:',
|
||||
error instanceof Error ? error.message : error,
|
||||
);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async function recoverRunFromDeliverables({
|
||||
runId,
|
||||
userId,
|
||||
@@ -2688,6 +2730,11 @@ export function createAgentRunGateway({
|
||||
if (validationInput && !enforcePageDataWorkflowValidation) {
|
||||
dispatchPageDataValidationObservation(validationInput);
|
||||
}
|
||||
await releaseDeliverableContracts({
|
||||
userId: row.user_id,
|
||||
deliverables,
|
||||
deliveryResult,
|
||||
});
|
||||
if (typeof observePersonalMemoryOnSuccess === 'function') {
|
||||
await observePersonalMemoryOnSuccess({
|
||||
userId: row.user_id,
|
||||
@@ -2754,6 +2801,54 @@ export function createAgentRunGateway({
|
||||
await finalizeSuccessfulRun(runId, row, recoverySessionId);
|
||||
return;
|
||||
}
|
||||
const cursorTakeover = resolveCursorPageTimeoutTakeover({
|
||||
row,
|
||||
error: err,
|
||||
toolGatewayStatus: toolGateway?.getStatus?.() ?? null,
|
||||
env: process.env,
|
||||
});
|
||||
if (cursorTakeover) {
|
||||
await appendEvent(runId, 'cursor_page_timeout_takeover', {
|
||||
timeoutMs: runTimeoutMs,
|
||||
executor: cursorTakeover.runOptions.requiredExecutor ?? null,
|
||||
});
|
||||
const takeoverRow = {
|
||||
...row,
|
||||
user_message_json: JSON.stringify(cursorTakeover.userMessage),
|
||||
};
|
||||
try {
|
||||
const execution = await runWithTimeout(
|
||||
runId,
|
||||
() => executeRun(takeoverRow, runId),
|
||||
);
|
||||
runExecutionContext.set(runId, execution);
|
||||
await finalizeSuccessfulRun(
|
||||
runId,
|
||||
takeoverRow,
|
||||
execution.sessionId,
|
||||
execution,
|
||||
);
|
||||
return;
|
||||
} catch (takeoverErr) {
|
||||
const takeoverLatest = await getRunById(runId);
|
||||
const takeoverSessionId =
|
||||
takeoverLatest?.agent_session_id ?? recoverySessionId;
|
||||
if (
|
||||
takeoverErr?.code !== 'IMAGE_GENERATION_REQUIRED_MISSING'
|
||||
&& await recoverRunFromDeliverables({
|
||||
runId,
|
||||
userId: row.user_id,
|
||||
sessionId: takeoverSessionId,
|
||||
err: takeoverErr,
|
||||
runStartedAtMs: takeoverLatest?.started_at ?? row.started_at ?? null,
|
||||
})
|
||||
) {
|
||||
await finalizeSuccessfulRun(runId, takeoverRow, takeoverSessionId);
|
||||
return;
|
||||
}
|
||||
err = takeoverErr;
|
||||
}
|
||||
}
|
||||
const rawMessage = err instanceof Error ? err.message : String(err);
|
||||
const alreadyEnriched = (Array.isArray(err?.suggestions) && err.suggestions.length > 0)
|
||||
|| /建议(/.test(rawMessage);
|
||||
|
||||
@@ -2126,6 +2126,59 @@ test('agent run succeeds when Finish is missing but session pages were already c
|
||||
);
|
||||
});
|
||||
|
||||
test('agent run recovery releases delivery contracts when Finish never arrives', async () => {
|
||||
const releaseCalls = [];
|
||||
const pool = createFakePool({
|
||||
sessionDeliverables: {
|
||||
'user-1:session-deliverable-release': [{
|
||||
page_id: 'page-shanghai',
|
||||
title: '上海一日游攻略',
|
||||
publication_id: 'pub-shanghai',
|
||||
publication_status: 'online',
|
||||
public_url: 'http://127.0.0.1:8081/MindSpace/user-1/public/shanghai-one-day-trip.html',
|
||||
workspace_relative_path: 'public/shanghai-one-day-trip.html',
|
||||
}],
|
||||
},
|
||||
});
|
||||
const gateway = createAgentRunGateway({
|
||||
pool,
|
||||
userAuth: {},
|
||||
tkmindProxy: {
|
||||
async startSessionForUser() {
|
||||
return { id: 'session-deliverable-release' };
|
||||
},
|
||||
async submitSessionReplyAndAwaitFinishForUser() {
|
||||
const err = new Error('agent run timed out after 900000ms');
|
||||
err.code = 'AGENT_RUN_TIMEOUT';
|
||||
throw err;
|
||||
},
|
||||
},
|
||||
releaseUserPageDeliveryContractsOnSuccess: async (input) => {
|
||||
releaseCalls.push(input);
|
||||
return input.relativePaths;
|
||||
},
|
||||
retryDelaysMs: [],
|
||||
});
|
||||
|
||||
const run = await gateway.createRun('user-1', {
|
||||
requestId: 'req-deliverable-release',
|
||||
userMessage: {
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我生成上海一日游攻略页面' }],
|
||||
metadata: {
|
||||
displayText: '帮我生成上海一日游攻略页面',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||
assert.equal(releaseCalls.length, 1);
|
||||
assert.deepEqual(releaseCalls[0], {
|
||||
userId: 'user-1',
|
||||
relativePaths: ['public/shanghai-one-day-trip.html'],
|
||||
});
|
||||
});
|
||||
|
||||
test('static page run succeeds when workspace fallback reports the current HTML path', async () => {
|
||||
let observedRunStartedAtMs = null;
|
||||
const pool = createFakePool({
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import {
|
||||
isGenericPageGenerationRequest,
|
||||
isPageGenerationIntent,
|
||||
} from './chat-skills.mjs';
|
||||
import {
|
||||
cursorExecutorEnabled,
|
||||
resolvePreferredCodeExecutor,
|
||||
} from './cursor-agent-launch.mjs';
|
||||
import { enforcePageGenerationCursorRuntime } from './cursor-page-routing.mjs';
|
||||
|
||||
function envFlag(value, fallback = false) {
|
||||
const raw = String(value ?? '').trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
return ['1', 'true', 'yes', 'on'].includes(raw);
|
||||
}
|
||||
|
||||
export function cursorPageTimeoutTakeoverEnabled(env = process.env) {
|
||||
return envFlag(env.MEMIND_CURSOR_PAGE_TIMEOUT_TAKEOVER, false);
|
||||
}
|
||||
|
||||
function extractRunDisplayText(row) {
|
||||
const message = row?.user_message_json;
|
||||
const parsed = typeof message === 'string'
|
||||
? (() => {
|
||||
try {
|
||||
return JSON.parse(message);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
})()
|
||||
: (message ?? {});
|
||||
const displayText = String(parsed?.metadata?.displayText ?? '').trim();
|
||||
if (displayText) return displayText;
|
||||
const content = parsed?.content;
|
||||
if (typeof content === 'string') return content.trim();
|
||||
if (Array.isArray(content)) {
|
||||
return content
|
||||
.filter((item) => item?.type === 'text')
|
||||
.map((item) => String(item.text ?? '').trim())
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function getRunOptionsFromMessage(userMessage) {
|
||||
const metadata = userMessage?.metadata;
|
||||
const runMetadata = metadata?.memindRun ?? metadata?.agentRun ?? {};
|
||||
return {
|
||||
toolMode: String(runMetadata?.toolMode ?? metadata?.toolMode ?? 'chat').trim().toLowerCase(),
|
||||
taskType: runMetadata?.taskType ?? metadata?.taskType ?? null,
|
||||
forceDeepReasoning: runMetadata?.forceDeepReasoning === true || metadata?.forceDeepReasoning === true,
|
||||
};
|
||||
}
|
||||
|
||||
function cursorExecutorAvailable(toolGatewayStatus, env = process.env) {
|
||||
if (!cursorExecutorEnabled(env)) return false;
|
||||
if (resolvePreferredCodeExecutor(env) !== 'cursor') return false;
|
||||
if (!toolGatewayStatus?.enabled) return false;
|
||||
const executors = Array.isArray(toolGatewayStatus.executors)
|
||||
? toolGatewayStatus.executors.map((item) => String(item).trim().toLowerCase())
|
||||
: [];
|
||||
return executors.includes('cursor');
|
||||
}
|
||||
|
||||
export function shouldRetryPageGenerationWithCursor({
|
||||
row,
|
||||
error = null,
|
||||
toolGatewayStatus = null,
|
||||
env = process.env,
|
||||
} = {}) {
|
||||
if (!cursorPageTimeoutTakeoverEnabled(env)) return false;
|
||||
if (String(error?.code ?? '') !== 'AGENT_RUN_TIMEOUT') return false;
|
||||
if (!cursorExecutorAvailable(toolGatewayStatus, env)) return false;
|
||||
|
||||
const userMessage = typeof row?.user_message_json === 'string'
|
||||
? (() => {
|
||||
try {
|
||||
return JSON.parse(row.user_message_json);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
})()
|
||||
: (row?.user_message_json ?? {});
|
||||
const runMetadata = userMessage?.metadata?.memindRun ?? userMessage?.metadata?.agentRun ?? {};
|
||||
if (runMetadata.pageCursorDefault === true || runMetadata.cursorTimeoutTakeover === true) {
|
||||
return false;
|
||||
}
|
||||
if (String(runMetadata.requiredExecutor ?? runMetadata.executor ?? '').trim().toLowerCase() === 'cursor') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const taskText = extractRunDisplayText(row);
|
||||
if (!isPageGenerationIntent(taskText) || isGenericPageGenerationRequest(taskText)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
export function buildCursorPageTimeoutTakeover(row, { env = process.env } = {}) {
|
||||
const userMessage = typeof row?.user_message_json === 'string'
|
||||
? (() => {
|
||||
try {
|
||||
return JSON.parse(row.user_message_json);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
})()
|
||||
: (row?.user_message_json ?? {});
|
||||
const runOptions = getRunOptionsFromMessage(userMessage);
|
||||
const enforced = enforcePageGenerationCursorRuntime(userMessage, {
|
||||
rawToolMode: runOptions.toolMode,
|
||||
taskType: runOptions.taskType,
|
||||
forceDeepReasoning: runOptions.forceDeepReasoning ?? true,
|
||||
env,
|
||||
channelEligible: true,
|
||||
policy: null,
|
||||
});
|
||||
if (!enforced.requiredExecutor) return null;
|
||||
|
||||
const metadata = enforced.userMessage?.metadata ?? {};
|
||||
const memindRun = metadata.memindRun ?? {};
|
||||
const updatedMessage = {
|
||||
...enforced.userMessage,
|
||||
metadata: {
|
||||
...metadata,
|
||||
memindRun: {
|
||||
...memindRun,
|
||||
cursorTimeoutTakeover: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
return {
|
||||
userMessage: updatedMessage,
|
||||
runOptions: {
|
||||
toolMode: enforced.rawToolMode,
|
||||
taskType: enforced.taskType,
|
||||
forceDeepReasoning: enforced.forceDeepReasoning,
|
||||
requiredExecutor: enforced.requiredExecutor,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveCursorPageTimeoutTakeover(input) {
|
||||
if (!shouldRetryPageGenerationWithCursor(input)) return null;
|
||||
return buildCursorPageTimeoutTakeover(input.row, { env: input.env });
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
import {
|
||||
buildCursorPageTimeoutTakeover,
|
||||
cursorPageTimeoutTakeoverEnabled,
|
||||
resolveCursorPageTimeoutTakeover,
|
||||
shouldRetryPageGenerationWithCursor,
|
||||
} from './cursor-page-timeout-takeover.mjs';
|
||||
|
||||
const enabledEnv = {
|
||||
MEMIND_CURSOR_EXECUTOR_ENABLED: '1',
|
||||
MEMIND_CURSOR_PAGE_TIMEOUT_TAKEOVER: '1',
|
||||
MEMIND_AIDER_SKILL_USE_CURSOR: '1',
|
||||
};
|
||||
|
||||
test('cursorPageTimeoutTakeoverEnabled respects env flag', () => {
|
||||
assert.equal(cursorPageTimeoutTakeoverEnabled({}), false);
|
||||
assert.equal(cursorPageTimeoutTakeoverEnabled(enabledEnv), true);
|
||||
});
|
||||
|
||||
test('shouldRetryPageGenerationWithCursor accepts page timeout without deliverables', () => {
|
||||
const row = {
|
||||
user_message_json: JSON.stringify({
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我生成上海一日游攻略页面' }],
|
||||
metadata: {
|
||||
displayText: '帮我生成上海一日游攻略页面',
|
||||
memindRun: { toolMode: 'chat' },
|
||||
},
|
||||
}),
|
||||
};
|
||||
assert.equal(
|
||||
shouldRetryPageGenerationWithCursor({
|
||||
row,
|
||||
error: { code: 'AGENT_RUN_TIMEOUT' },
|
||||
toolGatewayStatus: { enabled: true, executors: ['cursor'] },
|
||||
env: enabledEnv,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
test('shouldRetryPageGenerationWithCursor skips when deliverables already recovered', () => {
|
||||
const row = {
|
||||
user_message_json: JSON.stringify({
|
||||
metadata: {
|
||||
displayText: '帮我生成上海一日游攻略页面',
|
||||
memindRun: { pageCursorDefault: true },
|
||||
},
|
||||
}),
|
||||
};
|
||||
assert.equal(
|
||||
shouldRetryPageGenerationWithCursor({
|
||||
row,
|
||||
error: { code: 'AGENT_RUN_TIMEOUT' },
|
||||
toolGatewayStatus: { enabled: true, executors: ['cursor'] },
|
||||
env: enabledEnv,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('buildCursorPageTimeoutTakeover rewrites message for cursor code executor', () => {
|
||||
const row = {
|
||||
user_message_json: JSON.stringify({
|
||||
role: 'user',
|
||||
content: [{ type: 'text', text: '帮我生成上海一日游攻略页面' }],
|
||||
metadata: {
|
||||
displayText: '帮我生成上海一日游攻略页面',
|
||||
memindRun: { toolMode: 'chat' },
|
||||
},
|
||||
}),
|
||||
};
|
||||
const takeover = buildCursorPageTimeoutTakeover(row, { env: enabledEnv });
|
||||
assert.ok(takeover);
|
||||
assert.equal(takeover.runOptions.requiredExecutor, 'cursor');
|
||||
assert.equal(takeover.runOptions.toolMode, 'code');
|
||||
assert.equal(takeover.userMessage.metadata.memindRun.cursorTimeoutTakeover, true);
|
||||
assert.match(
|
||||
takeover.userMessage.content[0].text,
|
||||
/MindSpace 工作区生成或更新 public\/\*\.html 页面/,
|
||||
);
|
||||
});
|
||||
|
||||
test('resolveCursorPageTimeoutTakeover returns null for non-timeout errors', () => {
|
||||
const row = {
|
||||
user_message_json: JSON.stringify({
|
||||
metadata: { displayText: '帮我生成上海一日游攻略页面' },
|
||||
}),
|
||||
};
|
||||
assert.equal(
|
||||
resolveCursorPageTimeoutTakeover({
|
||||
row,
|
||||
error: { code: 'PUBLIC_PAGE_DELIVERABLE_MISSING' },
|
||||
toolGatewayStatus: { enabled: true, executors: ['cursor'] },
|
||||
env: enabledEnv,
|
||||
}),
|
||||
null,
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user