feat(orchestrator): observe Page Data validation
This commit is contained in:
+154
-3
@@ -111,6 +111,43 @@ function messageText(message) {
|
|||||||
.join('\n');
|
.join('\n');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isPageDataWorkflowRequested(userMessage, taskType = null) {
|
||||||
|
const metadata = userMessage?.metadata ?? {};
|
||||||
|
const runMetadata = metadata?.[RUN_METADATA_KEY] ?? metadata?.agentRun ?? {};
|
||||||
|
const normalizedTaskType = String(
|
||||||
|
taskType ?? runMetadata?.taskType ?? metadata?.taskType ?? '',
|
||||||
|
).trim().toLowerCase();
|
||||||
|
const selectedSkill = String(
|
||||||
|
runMetadata?.selectedChatSkill ?? metadata?.selectedChatSkill ?? '',
|
||||||
|
).trim();
|
||||||
|
const displayText = String(metadata?.displayText ?? '').trim()
|
||||||
|
|| deriveUserFacingText(messageText(userMessage));
|
||||||
|
return runMetadata?.pgRequired === true
|
||||||
|
|| runMetadata?.pageDataAiderWorkflow === true
|
||||||
|
|| normalizedTaskType === 'page_data_dev'
|
||||||
|
|| normalizedTaskType === 'page_data_dev_complex'
|
||||||
|
|| selectedSkill === 'page-data-collect'
|
||||||
|
|| isPageDataIntent(displayText);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pageDataFailureCheckId(code) {
|
||||||
|
switch (String(code ?? '').trim().toUpperCase()) {
|
||||||
|
case 'PAGE_DATA_DELIVERY_FAILED':
|
||||||
|
return 'page_data_binding';
|
||||||
|
case 'PAGE_DATA_DELIVERABLE_MISSING':
|
||||||
|
return 'page_data_deliverable';
|
||||||
|
case 'DELIVERABLE_DATA_STORAGE_FORBIDDEN':
|
||||||
|
return 'page_data_storage_policy';
|
||||||
|
case 'REQUIRED_REVIEW_EXECUTOR_MISMATCH':
|
||||||
|
case 'REQUIRED_EXECUTOR_UNAVAILABLE':
|
||||||
|
return 'independent_review';
|
||||||
|
case 'TOOL_GATEWAY_VALIDATION_FAILED':
|
||||||
|
return 'code_validation';
|
||||||
|
default:
|
||||||
|
return 'agent_run_completion';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 12_000 } = {}) {
|
function buildFreshSessionContext(conversation, { maxMessages = 16, maxChars = 12_000 } = {}) {
|
||||||
const visible = [];
|
const visible = [];
|
||||||
for (const message of Array.isArray(conversation) ? conversation : []) {
|
for (const message of Array.isArray(conversation) ? conversation : []) {
|
||||||
@@ -394,6 +431,7 @@ function getRunOptionsFromMessage(userMessage) {
|
|||||||
? String(runMetadata.reviewExecutor).trim().toLowerCase()
|
? String(runMetadata.reviewExecutor).trim().toLowerCase()
|
||||||
: null,
|
: null,
|
||||||
pageDataAiderWorkflow: runMetadata?.pageDataAiderWorkflow === true,
|
pageDataAiderWorkflow: runMetadata?.pageDataAiderWorkflow === true,
|
||||||
|
pgRequired: runMetadata?.pgRequired === true,
|
||||||
forceDeepReasoning: runMetadata?.forceDeepReasoning === true || metadata?.forceDeepReasoning === true,
|
forceDeepReasoning: runMetadata?.forceDeepReasoning === true || metadata?.forceDeepReasoning === true,
|
||||||
validation: normalizeToolGatewayValidation(runMetadata?.validation ?? metadata?.toolGatewayValidation),
|
validation: normalizeToolGatewayValidation(runMetadata?.validation ?? metadata?.toolGatewayValidation),
|
||||||
sessionMessageCount: normalizeSessionMessageCount(
|
sessionMessageCount: normalizeSessionMessageCount(
|
||||||
@@ -475,6 +513,7 @@ export function createAgentRunGateway({
|
|||||||
syncUserPagesOnSuccess = null,
|
syncUserPagesOnSuccess = null,
|
||||||
observePersonalMemoryOnSuccess = null,
|
observePersonalMemoryOnSuccess = null,
|
||||||
observeWorkflowRun = null,
|
observeWorkflowRun = null,
|
||||||
|
observeWorkflowValidation = null,
|
||||||
isSessionExternallyBusy = null,
|
isSessionExternallyBusy = null,
|
||||||
validateRunDeliverables = null,
|
validateRunDeliverables = null,
|
||||||
quiesceSessionOnTerminal = null,
|
quiesceSessionOnTerminal = null,
|
||||||
@@ -637,9 +676,78 @@ export function createAgentRunGateway({
|
|||||||
logger: console,
|
logger: console,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const validationDispatcher = createShadowObservationDispatcher({
|
||||||
|
observe: observeWorkflowValidation,
|
||||||
|
maxConcurrent: maxConcurrentShadowObservations,
|
||||||
|
maxQueued: maxQueuedShadowObservations,
|
||||||
|
onCompleted: async (input, result, context) => {
|
||||||
|
if (!result?.observed) {
|
||||||
|
await appendEvent(input.runId, 'workflow_validation_observation_skipped', {
|
||||||
|
reason: result?.reason ?? 'shadow_validation_not_selected',
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await appendEvent(input.runId, 'workflow_validation_observation_completed', {
|
||||||
|
engine: result.engine ?? 'langgraph',
|
||||||
|
mode: result.mode ?? 'shadow',
|
||||||
|
configVersion: result.configVersion ?? null,
|
||||||
|
kind: result.validation?.kind ?? 'page-data-delivery',
|
||||||
|
verdict: result.validation?.verdict ?? null,
|
||||||
|
latencyMs: Math.max(0, nowMs() - context.enqueuedAt),
|
||||||
|
});
|
||||||
|
},
|
||||||
|
onFailed: async (input, error, context) => {
|
||||||
|
await appendEvent(input.runId, 'workflow_validation_observation_failed', {
|
||||||
|
code: String(error?.code ?? 'WORKFLOW_VALIDATION_OBSERVATION_FAILED').slice(0, 128),
|
||||||
|
message: String(error instanceof Error ? error.message : error).slice(0, 1000),
|
||||||
|
latencyMs: Math.max(0, nowMs() - context.enqueuedAt),
|
||||||
|
}).catch(() => {});
|
||||||
|
},
|
||||||
|
onSkipped: async (input, context) => {
|
||||||
|
await appendEvent(input.runId, 'workflow_validation_observation_skipped', {
|
||||||
|
reason: context.reason,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
logger: console,
|
||||||
|
});
|
||||||
|
|
||||||
function dispatchShadowObservation(input) {
|
function dispatchShadowObservation(input) {
|
||||||
if (input.toolMode !== 'code') return;
|
const pageDataRequired = isPageDataWorkflowRequested(input.userMessage, input.taskType);
|
||||||
shadowDispatcher.dispatch(input);
|
if (input.toolMode !== 'code' && !pageDataRequired) return;
|
||||||
|
shadowDispatcher.dispatch({
|
||||||
|
...input,
|
||||||
|
taskType: input.taskType || (pageDataRequired ? 'page_data_dev' : null),
|
||||||
|
pageDataRequired,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function dispatchPageDataValidationObservation({
|
||||||
|
row,
|
||||||
|
runId,
|
||||||
|
checks,
|
||||||
|
metrics = null,
|
||||||
|
}) {
|
||||||
|
const userMessage = parseDbJsonColumn(row?.user_message_json, {}) ?? {};
|
||||||
|
const runOptions = getRunOptionsFromMessage(userMessage);
|
||||||
|
if (!isPageDataWorkflowRequested(userMessage, runOptions.taskType)) return false;
|
||||||
|
return validationDispatcher.dispatch({
|
||||||
|
runId,
|
||||||
|
requestId: row?.request_id ?? null,
|
||||||
|
userId: row?.user_id ?? null,
|
||||||
|
workflowName: 'code-run-v1',
|
||||||
|
observation: {
|
||||||
|
idempotencyKey: `${runId}:page-data-delivery:v1`,
|
||||||
|
taskType: runOptions.taskType || 'page_data_dev',
|
||||||
|
required: true,
|
||||||
|
checks,
|
||||||
|
metrics: {
|
||||||
|
pageCount: Number(metrics?.pageCount ?? 0),
|
||||||
|
publicationCount: Number(metrics?.publicationCount ?? 0),
|
||||||
|
},
|
||||||
|
source: 'portal-agent-run',
|
||||||
|
observedAt: nowMs(),
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function appendRunSnapshot(runId) {
|
async function appendRunSnapshot(runId) {
|
||||||
@@ -1464,6 +1572,8 @@ export function createAgentRunGateway({
|
|||||||
}, { expectedStatus: 'running' });
|
}, { expectedStatus: 'running' });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
const userMessage = parseDbJsonColumn(row.user_message_json, {}) ?? {};
|
||||||
|
const runOptions = getRunOptionsFromMessage(userMessage);
|
||||||
assertRequiredImageGenerationCompleted(row, routing, toolEvidence);
|
assertRequiredImageGenerationCompleted(row, routing, toolEvidence);
|
||||||
// `row` was loaded before this worker claimed the run, so its started_at
|
// `row` was loaded before this worker claimed the run, so its started_at
|
||||||
// can still be null. Refresh it before scoping workspace files to the
|
// can still be null. Refresh it before scoping workspace files to the
|
||||||
@@ -1490,7 +1600,8 @@ export function createAgentRunGateway({
|
|||||||
}
|
}
|
||||||
const runDisplayText = extractRunDisplayText(row);
|
const runDisplayText = extractRunDisplayText(row);
|
||||||
const selectedSkill = selectedRunSkill(row);
|
const selectedSkill = selectedRunSkill(row);
|
||||||
const pageDataIntent = isPageDataIntent(runDisplayText)
|
const pageDataIntent = isPageDataWorkflowRequested(userMessage, runOptions.taskType)
|
||||||
|
|| isPageDataIntent(runDisplayText)
|
||||||
|| selectedSkill === 'page-data-collect'
|
|| selectedSkill === 'page-data-collect'
|
||||||
|| routing?.suggestedSkill === 'page-data-collect';
|
|| routing?.suggestedSkill === 'page-data-collect';
|
||||||
const pageGenerationIntent = isPageGenerationIntent(runDisplayText)
|
const pageGenerationIntent = isPageGenerationIntent(runDisplayText)
|
||||||
@@ -1557,6 +1668,35 @@ export function createAgentRunGateway({
|
|||||||
error_message: null,
|
error_message: null,
|
||||||
}, { expectedStatus: 'running' });
|
}, { expectedStatus: 'running' });
|
||||||
if (!marked) return false;
|
if (!marked) return false;
|
||||||
|
if (pageDataIntent) {
|
||||||
|
dispatchPageDataValidationObservation({
|
||||||
|
row,
|
||||||
|
runId,
|
||||||
|
checks: [
|
||||||
|
{ id: 'agent_run_completion', status: 'passed' },
|
||||||
|
{
|
||||||
|
id: 'page_data_binding',
|
||||||
|
status: typeof syncUserPagesOnSuccess === 'function' ? 'passed' : 'skipped',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'page_data_storage_policy',
|
||||||
|
status: typeof validateRunDeliverables === 'function' ? 'passed' : 'skipped',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'page_data_deliverable',
|
||||||
|
status: requiresPageDeliverable ? 'passed' : 'skipped',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'independent_review',
|
||||||
|
status: runOptions.reviewExecutor ? 'passed' : 'skipped',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
metrics: {
|
||||||
|
pageCount: deliverables?.pageCount ?? 0,
|
||||||
|
publicationCount: deliverables?.publicationCount ?? 0,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
if (typeof observePersonalMemoryOnSuccess === 'function') {
|
if (typeof observePersonalMemoryOnSuccess === 'function') {
|
||||||
await observePersonalMemoryOnSuccess({
|
await observePersonalMemoryOnSuccess({
|
||||||
userId: row.user_id,
|
userId: row.user_id,
|
||||||
@@ -1639,6 +1779,17 @@ export function createAgentRunGateway({
|
|||||||
}
|
}
|
||||||
: {}),
|
: {}),
|
||||||
}, { expectedStatus: 'running' });
|
}, { expectedStatus: 'running' });
|
||||||
|
if (!retryable) {
|
||||||
|
dispatchPageDataValidationObservation({
|
||||||
|
row,
|
||||||
|
runId,
|
||||||
|
checks: [{
|
||||||
|
id: pageDataFailureCheckId(err?.code),
|
||||||
|
status: 'failed',
|
||||||
|
codes: [String(err?.code ?? 'AGENT_RUN_FAILED')],
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
}
|
||||||
if (retryable && autoDispatch) {
|
if (retryable && autoDispatch) {
|
||||||
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
setTimeout(() => dispatchRun(runId), retryDelaysMs[nextAttempt - 1]);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -462,6 +462,153 @@ test('agent run creation stores code tool metadata in the queued message', async
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('explicit Page Data selection enters Orchestrator Shadow even when the run starts in chat mode', async () => {
|
||||||
|
const pool = createFakePool();
|
||||||
|
const observed = [];
|
||||||
|
const gateway = createAgentRunGateway({
|
||||||
|
pool,
|
||||||
|
userAuth: {},
|
||||||
|
tkmindProxy: {},
|
||||||
|
autoDispatch: false,
|
||||||
|
observeWorkflowRun: async (input) => {
|
||||||
|
observed.push(input);
|
||||||
|
return { observed: false, reason: 'test-only' };
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const run = await gateway.createRun('user-page-data-shadow', {
|
||||||
|
requestId: 'req-page-data-shadow',
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '解释一下这个数据空间' }],
|
||||||
|
metadata: {
|
||||||
|
displayText: '解释一下这个数据空间',
|
||||||
|
memindRun: { pgRequired: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
toolMode: 'chat',
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor(() => observed.length === 1);
|
||||||
|
assert.equal(observed[0].runId, run.id);
|
||||||
|
assert.equal(observed[0].pageDataRequired, true);
|
||||||
|
assert.equal(observed[0].taskType, 'page_data_dev');
|
||||||
|
assert.equal(pool.runs.get(run.id).status, 'queued');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('successful Page Data runs send bounded validation evidence without changing Native execution', async () => {
|
||||||
|
const pool = createFakePool();
|
||||||
|
const validationInputs = [];
|
||||||
|
const submitted = [];
|
||||||
|
const gateway = createAgentRunGateway({
|
||||||
|
pool,
|
||||||
|
userAuth: {},
|
||||||
|
tkmindProxy: {
|
||||||
|
async startSessionForUser() {
|
||||||
|
return { id: 'session-page-data-validation' };
|
||||||
|
},
|
||||||
|
async submitSessionReplyForUser(userId, sessionId, requestId, userMessage) {
|
||||||
|
submitted.push({ userId, sessionId, requestId, userMessage });
|
||||||
|
},
|
||||||
|
},
|
||||||
|
observeWorkflowRun: async () => ({ observed: true }),
|
||||||
|
observeWorkflowValidation: async (input) => {
|
||||||
|
validationInputs.push(input);
|
||||||
|
return {
|
||||||
|
observed: true,
|
||||||
|
engine: 'langgraph',
|
||||||
|
mode: 'shadow',
|
||||||
|
validation: {
|
||||||
|
kind: 'page-data-delivery',
|
||||||
|
verdict: 'inconclusive',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
retryDelaysMs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const run = await gateway.createRun('user-page-data-validation', {
|
||||||
|
requestId: 'req-page-data-validation',
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '解释 Page Data API 的用途' }],
|
||||||
|
metadata: {
|
||||||
|
displayText: '解释 Page Data API 的用途',
|
||||||
|
memindRun: { pgRequired: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
toolMode: 'chat',
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||||
|
await waitFor(() => validationInputs.length === 1);
|
||||||
|
assert.equal(submitted.length, 1);
|
||||||
|
assert.equal(validationInputs[0].runId, run.id);
|
||||||
|
assert.equal(validationInputs[0].observation.required, true);
|
||||||
|
assert.deepEqual(
|
||||||
|
validationInputs[0].observation.checks.map((check) => [check.id, check.status]),
|
||||||
|
[
|
||||||
|
['agent_run_completion', 'passed'],
|
||||||
|
['page_data_binding', 'skipped'],
|
||||||
|
['page_data_storage_policy', 'skipped'],
|
||||||
|
['page_data_deliverable', 'skipped'],
|
||||||
|
['independent_review', 'skipped'],
|
||||||
|
],
|
||||||
|
);
|
||||||
|
const projected = JSON.stringify(validationInputs[0].observation);
|
||||||
|
assert.equal(projected.includes('解释 Page Data API'), false);
|
||||||
|
assert.equal(pool.runs.get(run.id).status, 'succeeded');
|
||||||
|
await waitFor(() => pool.events.some(
|
||||||
|
(event) => event.runId === run.id
|
||||||
|
&& event.eventType === 'workflow_validation_observation_completed',
|
||||||
|
));
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Page Data validation observation failure stays isolated from the Native result', async () => {
|
||||||
|
const pool = createFakePool();
|
||||||
|
const gateway = createAgentRunGateway({
|
||||||
|
pool,
|
||||||
|
userAuth: {},
|
||||||
|
tkmindProxy: {
|
||||||
|
async startSessionForUser() {
|
||||||
|
return { id: 'session-page-data-validation-failure' };
|
||||||
|
},
|
||||||
|
async submitSessionReplyForUser() {},
|
||||||
|
},
|
||||||
|
observeWorkflowRun: async () => ({ observed: true }),
|
||||||
|
observeWorkflowValidation: async () => {
|
||||||
|
throw Object.assign(new Error('validation observer unavailable'), {
|
||||||
|
code: 'VALIDATION_OBSERVER_UNAVAILABLE',
|
||||||
|
});
|
||||||
|
},
|
||||||
|
retryDelaysMs: [],
|
||||||
|
});
|
||||||
|
|
||||||
|
const run = await gateway.createRun('user-page-data-validation-failure', {
|
||||||
|
requestId: 'req-page-data-validation-failure',
|
||||||
|
userMessage: {
|
||||||
|
role: 'user',
|
||||||
|
content: [{ type: 'text', text: '解释 Page Data API' }],
|
||||||
|
metadata: {
|
||||||
|
displayText: '解释 Page Data API',
|
||||||
|
memindRun: { pgRequired: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
await waitFor(() => pool.runs.get(run.id)?.status === 'succeeded');
|
||||||
|
await waitFor(() => pool.events.some(
|
||||||
|
(event) => event.runId === run.id
|
||||||
|
&& event.eventType === 'workflow_validation_observation_failed',
|
||||||
|
));
|
||||||
|
assert.equal(pool.runs.get(run.id).status, 'succeeded');
|
||||||
|
const failure = JSON.parse(pool.events.find(
|
||||||
|
(event) => event.runId === run.id
|
||||||
|
&& event.eventType === 'workflow_validation_observation_failed',
|
||||||
|
).dataJson);
|
||||||
|
assert.equal(failure.code, 'VALIDATION_OBSERVER_UNAVAILABLE');
|
||||||
|
});
|
||||||
|
|
||||||
test('agent run starts a session and marks submitted reply as succeeded', async () => {
|
test('agent run starts a session and marks submitted reply as succeeded', async () => {
|
||||||
const pool = createFakePool();
|
const pool = createFakePool();
|
||||||
const submitted = [];
|
const submitted = [];
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ The framework-neutral contracts are:
|
|||||||
- `executor-job-request-v1`
|
- `executor-job-request-v1`
|
||||||
- `executor-job-state-v1`
|
- `executor-job-state-v1`
|
||||||
- `executor-dispatch-decision-v1`
|
- `executor-dispatch-decision-v1`
|
||||||
|
- `page-data-validation-observation-v1`
|
||||||
|
|
||||||
The Phase 3.2 Executor Gateway owns the adapter registry and executor job state
|
The Phase 3.2 Executor Gateway owns the adapter registry and executor job state
|
||||||
transition boundary. Requests contain workspace/artifact references,
|
transition boundary. Requests contain workspace/artifact references,
|
||||||
@@ -155,6 +156,7 @@ The implemented internal API is:
|
|||||||
```text
|
```text
|
||||||
POST /v1/runs
|
POST /v1/runs
|
||||||
GET /v1/runs/:id
|
GET /v1/runs/:id
|
||||||
|
POST /v1/runs/:id/validation-observations
|
||||||
POST /v1/runs/:id/resume
|
POST /v1/runs/:id/resume
|
||||||
POST /v1/runs/:id/cancel
|
POST /v1/runs/:id/cancel
|
||||||
DELETE /v1/runs/:id
|
DELETE /v1/runs/:id
|
||||||
@@ -177,6 +179,16 @@ through `DELETE /v1/runs/:id`; deletion removes the LangGraph thread and its
|
|||||||
linked terminal Executor Job, whose events cascade in PostgreSQL. Active or
|
linked terminal Executor Job, whose events cascade in PostgreSQL. Active or
|
||||||
waiting runs fail deletion with `409`.
|
waiting runs fail deletion with `409`.
|
||||||
|
|
||||||
|
Page Data code runs may append a post-run
|
||||||
|
`page-data-validation-observation-v1` after the Native delivery guards finish.
|
||||||
|
The observation contains only bounded check identifiers, pass/fail/skipped
|
||||||
|
states, error codes, and aggregate page/publication counts. User prompts, code,
|
||||||
|
workspace paths, dataset contents, credentials, and database identifiers must
|
||||||
|
not cross this boundary. The Orchestrator derives and checkpoints a
|
||||||
|
`passed`/`failed`/`inconclusive` verdict and emits
|
||||||
|
`workflow_validation_observed`. In the initial observation phase this verdict
|
||||||
|
does not mutate the Native run result or publication state.
|
||||||
|
|
||||||
The graph keeps three deterministic control-plane nodes:
|
The graph keeps three deterministic control-plane nodes:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
|
|||||||
@@ -460,6 +460,25 @@ export type OrchestratorShadowRunDetail = {
|
|||||||
phase?: string;
|
phase?: string;
|
||||||
plan?: Record<string, unknown>;
|
plan?: Record<string, unknown>;
|
||||||
result?: Record<string, unknown>;
|
result?: Record<string, unknown>;
|
||||||
|
validation?: {
|
||||||
|
version?: string;
|
||||||
|
kind?: string;
|
||||||
|
taskType?: string | null;
|
||||||
|
required?: boolean;
|
||||||
|
verdict?: 'passed' | 'failed' | 'inconclusive';
|
||||||
|
checks?: Array<{
|
||||||
|
id?: string;
|
||||||
|
status?: 'passed' | 'failed' | 'skipped';
|
||||||
|
codes?: string[];
|
||||||
|
}>;
|
||||||
|
metrics?: {
|
||||||
|
pageCount?: number;
|
||||||
|
publicationCount?: number;
|
||||||
|
};
|
||||||
|
source?: string;
|
||||||
|
dataPolicy?: string;
|
||||||
|
observedAt?: number;
|
||||||
|
} | null;
|
||||||
} | null;
|
} | null;
|
||||||
events: Array<{
|
events: Array<{
|
||||||
sequence?: number;
|
sequence?: number;
|
||||||
|
|||||||
@@ -328,6 +328,19 @@ export function OrchestratorShadowPanel() {
|
|||||||
{selected.remote.executorJob.error.message}
|
{selected.remote.executorJob.error.message}
|
||||||
</p>
|
</p>
|
||||||
) : null}
|
) : null}
|
||||||
|
{selected.remote.state?.validation ? (
|
||||||
|
<p
|
||||||
|
className={selected.remote.state.validation.verdict === 'failed' ? 'alert' : undefined}
|
||||||
|
style={{ color: selected.remote.state.validation.verdict === 'failed' ? undefined : '#4f6258' }}
|
||||||
|
>
|
||||||
|
Page Data 验证:
|
||||||
|
{selected.remote.state.validation.verdict ?? 'unknown'}
|
||||||
|
{' · '}
|
||||||
|
{selected.remote.state.validation.checks?.length ?? 0} 项检查
|
||||||
|
{' · '}
|
||||||
|
{selected.remote.state.validation.dataPolicy ?? 'unknown-policy'}
|
||||||
|
</p>
|
||||||
|
) : null}
|
||||||
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))', gap: 12 }}>
|
<div style={{ display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))', gap: 12 }}>
|
||||||
<div>
|
<div>
|
||||||
<strong>LangGraph checkpoint</strong>
|
<strong>LangGraph checkpoint</strong>
|
||||||
|
|||||||
@@ -329,6 +329,8 @@ export function bootstrapPortalGatewayServices({
|
|||||||
sessionSnapshotService,
|
sessionSnapshotService,
|
||||||
conversationMemoryService,
|
conversationMemoryService,
|
||||||
observeWorkflowRun: workflowShadowObserver,
|
observeWorkflowRun: workflowShadowObserver,
|
||||||
|
observeWorkflowValidation:
|
||||||
|
workflowShadowObserver?.observeValidation ?? null,
|
||||||
observePersonalMemoryOnSuccess: async ({
|
observePersonalMemoryOnSuccess: async ({
|
||||||
userId,
|
userId,
|
||||||
sessionId,
|
sessionId,
|
||||||
|
|||||||
@@ -167,6 +167,7 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
|
|||||||
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
assert.equal(captured.agentOptions.maxConcurrentRuns, 3);
|
||||||
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
assert.equal(captured.agentOptions.runTimeoutMs, 9000);
|
||||||
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
assert.equal(captured.agentOptions.observeWorkflowRun, null);
|
||||||
|
assert.equal(captured.agentOptions.observeWorkflowValidation, null);
|
||||||
assert.equal(captured.agentOptions.maxConcurrentShadowObservations, 2);
|
assert.equal(captured.agentOptions.maxConcurrentShadowObservations, 2);
|
||||||
assert.equal(captured.agentOptions.maxQueuedShadowObservations, 100);
|
assert.equal(captured.agentOptions.maxQueuedShadowObservations, 100);
|
||||||
});
|
});
|
||||||
@@ -174,6 +175,8 @@ test('preserves Proxy, Tool, and Agent gateway wiring', () => {
|
|||||||
test('wires Shadow observation only behind the explicit environment gate', () => {
|
test('wires Shadow observation only behind the explicit environment gate', () => {
|
||||||
const configService = { id: 'orchestrator-config' };
|
const configService = { id: 'orchestrator-config' };
|
||||||
const observer = async () => ({ observed: false });
|
const observer = async () => ({ observed: false });
|
||||||
|
const validationObserver = async () => ({ observed: false });
|
||||||
|
observer.observeValidation = validationObserver;
|
||||||
let receivedObserverOptions = null;
|
let receivedObserverOptions = null;
|
||||||
const setup = createSetup({
|
const setup = createSetup({
|
||||||
env: {
|
env: {
|
||||||
@@ -197,6 +200,7 @@ test('wires Shadow observation only behind the explicit environment gate', () =>
|
|||||||
assert.equal(receivedObserverOptions.configService, configService);
|
assert.equal(receivedObserverOptions.configService, configService);
|
||||||
assert.equal(receivedObserverOptions.serviceToken, 'shadow-token');
|
assert.equal(receivedObserverOptions.serviceToken, 'shadow-token');
|
||||||
assert.equal(agentOptions.observeWorkflowRun, observer);
|
assert.equal(agentOptions.observeWorkflowRun, observer);
|
||||||
|
assert.equal(agentOptions.observeWorkflowValidation, validationObserver);
|
||||||
assert.equal(agentOptions.maxConcurrentShadowObservations, 4);
|
assert.equal(agentOptions.maxConcurrentShadowObservations, 4);
|
||||||
assert.equal(agentOptions.maxQueuedShadowObservations, 25);
|
assert.equal(agentOptions.maxQueuedShadowObservations, 25);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -134,6 +134,19 @@ export function createOrchestratorApp({
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.post('/v1/runs/:runId/validation-observations', async (request, response, next) => {
|
||||||
|
try {
|
||||||
|
const state = await runtime.recordValidationObservation(
|
||||||
|
request.params.runId,
|
||||||
|
request.body,
|
||||||
|
);
|
||||||
|
if (!state) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
|
||||||
|
return response.json(state);
|
||||||
|
} catch (error) {
|
||||||
|
return next(error);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
app.post('/v1/runs/:runId/resume', async (request, response, next) => {
|
app.post('/v1/runs/:runId/resume', async (request, response, next) => {
|
||||||
try {
|
try {
|
||||||
const state = await runtime.resume(request.params.runId, request.body);
|
const state = await runtime.resume(request.params.runId, request.body);
|
||||||
|
|||||||
@@ -76,12 +76,38 @@ test('orchestrator HTTP API exposes health and authenticated run endpoints', asy
|
|||||||
assert.equal(created.status, 202);
|
assert.equal(created.status, 202);
|
||||||
assert.equal((await created.json()).status, 'succeeded');
|
assert.equal((await created.json()).status, 'succeeded');
|
||||||
|
|
||||||
|
const validation = await fetch(
|
||||||
|
`${server.baseUrl}/v1/runs/http-shadow-1/validation-observations`,
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
authorization: 'Bearer test-service-token',
|
||||||
|
'content-type': 'application/json',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
idempotencyKey: 'http-shadow-1:page-data-delivery:v1',
|
||||||
|
taskType: 'page_data_dev',
|
||||||
|
required: true,
|
||||||
|
checks: [
|
||||||
|
{ id: 'agent_run_completion', status: 'passed' },
|
||||||
|
{ id: 'page_data_binding', status: 'passed' },
|
||||||
|
{ id: 'page_data_storage_policy', status: 'passed' },
|
||||||
|
],
|
||||||
|
metrics: { pageCount: 1, publicationCount: 1 },
|
||||||
|
observedAt: 123,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(validation.status, 200);
|
||||||
|
assert.equal((await validation.json()).validation.verdict, 'passed');
|
||||||
|
|
||||||
const events = await fetch(`${server.baseUrl}/v1/runs/http-shadow-1/events?after=1`, {
|
const events = await fetch(`${server.baseUrl}/v1/runs/http-shadow-1/events?after=1`, {
|
||||||
headers: { authorization: 'Bearer test-service-token' },
|
headers: { authorization: 'Bearer test-service-token' },
|
||||||
});
|
});
|
||||||
const eventBody = await events.json();
|
const eventBody = await events.json();
|
||||||
assert.equal(events.status, 200);
|
assert.equal(events.status, 200);
|
||||||
assert.deepEqual(eventBody.events.map((event) => event.sequence), [2, 3]);
|
assert.deepEqual(eventBody.events.map((event) => event.sequence), [2, 3, 4]);
|
||||||
|
assert.equal(eventBody.events.at(-1).type, 'workflow_validation_observed');
|
||||||
|
|
||||||
const jobId = 'http-shadow-1:executor-preview';
|
const jobId = 'http-shadow-1:executor-preview';
|
||||||
const executorJob = await fetch(
|
const executorJob = await fetch(
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ const ShadowState = Annotation.Root({
|
|||||||
reducer: (_current, update) => update,
|
reducer: (_current, update) => update,
|
||||||
default: () => null,
|
default: () => null,
|
||||||
}),
|
}),
|
||||||
|
validation: Annotation({
|
||||||
|
reducer: (_current, update) => update,
|
||||||
|
default: () => null,
|
||||||
|
}),
|
||||||
events: Annotation({
|
events: Annotation({
|
||||||
reducer: (current, update) => [...current, ...update],
|
reducer: (current, update) => [...current, ...update],
|
||||||
default: () => [],
|
default: () => [],
|
||||||
|
|||||||
@@ -16,8 +16,18 @@ export const DEFAULT_ORCHESTRATED_WORKFLOWS = Object.freeze([
|
|||||||
'code-run-v1',
|
'code-run-v1',
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
export const PAGE_DATA_VALIDATION_OBSERVATION_VERSION =
|
||||||
|
'page-data-validation-observation-v1';
|
||||||
|
|
||||||
const ENGINE_ID_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/;
|
const ENGINE_ID_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/;
|
||||||
const WORKFLOW_NAME_PATTERN = /^[a-z][a-z0-9_-]{0,127}$/;
|
const WORKFLOW_NAME_PATTERN = /^[a-z][a-z0-9_-]{0,127}$/;
|
||||||
|
const VALIDATION_CHECK_ID_PATTERN = /^[a-z][a-z0-9_-]{0,127}$/;
|
||||||
|
const VALIDATION_CHECK_STATUSES = new Set(['passed', 'failed', 'skipped']);
|
||||||
|
const REQUIRED_PAGE_DATA_CHECKS = Object.freeze([
|
||||||
|
'agent_run_completion',
|
||||||
|
'page_data_binding',
|
||||||
|
'page_data_storage_policy',
|
||||||
|
]);
|
||||||
|
|
||||||
function normalizeIdentifier(value, pattern, fallback) {
|
function normalizeIdentifier(value, pattern, fallback) {
|
||||||
const normalized = String(value ?? '').trim().toLowerCase();
|
const normalized = String(value ?? '').trim().toLowerCase();
|
||||||
@@ -98,6 +108,82 @@ export function buildRunEvent({
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validationContractError(message) {
|
||||||
|
const error = new Error(message);
|
||||||
|
error.code = 'WORKFLOW_VALIDATION_INVALID';
|
||||||
|
error.status = 422;
|
||||||
|
return error;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeValidationCount(value) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
if (!Number.isFinite(parsed) || parsed < 0) return 0;
|
||||||
|
return Math.min(1_000_000, Math.floor(parsed));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizePageDataValidationObservation(input = {}) {
|
||||||
|
const source = input && typeof input === 'object' && !Array.isArray(input) ? input : {};
|
||||||
|
const idempotencyKey = String(source.idempotencyKey ?? '').trim().slice(0, 200);
|
||||||
|
if (!idempotencyKey) {
|
||||||
|
throw validationContractError('Page Data validation observation requires idempotencyKey');
|
||||||
|
}
|
||||||
|
if (!Array.isArray(source.checks) || source.checks.length === 0) {
|
||||||
|
throw validationContractError('Page Data validation observation requires checks');
|
||||||
|
}
|
||||||
|
const checks = source.checks.slice(0, 32).map((item) => {
|
||||||
|
const check = item && typeof item === 'object' && !Array.isArray(item) ? item : {};
|
||||||
|
const id = normalizeIdentifier(check.id, VALIDATION_CHECK_ID_PATTERN, '');
|
||||||
|
const status = String(check.status ?? '').trim().toLowerCase();
|
||||||
|
if (!id || !VALIDATION_CHECK_STATUSES.has(status)) {
|
||||||
|
throw validationContractError('Page Data validation observation contains an invalid check');
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
status,
|
||||||
|
codes: normalizeStringList(check.codes, {
|
||||||
|
limit: 32,
|
||||||
|
itemLimit: 128,
|
||||||
|
normalize: (value) => String(value ?? '').trim().toUpperCase(),
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const byId = new Map(checks.map((check) => [check.id, check]));
|
||||||
|
const failed = checks.some((check) => check.status === 'failed');
|
||||||
|
const required = source.required === true;
|
||||||
|
const requiredChecksPassed = REQUIRED_PAGE_DATA_CHECKS.every(
|
||||||
|
(id) => byId.get(id)?.status === 'passed',
|
||||||
|
);
|
||||||
|
const verdict = failed
|
||||||
|
? 'failed'
|
||||||
|
: required && !requiredChecksPassed
|
||||||
|
? 'inconclusive'
|
||||||
|
: checks.some((check) => check.status === 'passed')
|
||||||
|
? 'passed'
|
||||||
|
: 'inconclusive';
|
||||||
|
const taskType = normalizeWorkflowName(source.taskType, '');
|
||||||
|
const observedAt = Number(source.observedAt);
|
||||||
|
return {
|
||||||
|
version: PAGE_DATA_VALIDATION_OBSERVATION_VERSION,
|
||||||
|
kind: 'page-data-delivery',
|
||||||
|
idempotencyKey,
|
||||||
|
taskType: taskType || null,
|
||||||
|
required,
|
||||||
|
verdict,
|
||||||
|
checks,
|
||||||
|
metrics: {
|
||||||
|
pageCount: normalizeValidationCount(source.metrics?.pageCount),
|
||||||
|
publicationCount: normalizeValidationCount(source.metrics?.publicationCount),
|
||||||
|
},
|
||||||
|
source: normalizeIdentifier(
|
||||||
|
source.source,
|
||||||
|
VALIDATION_CHECK_ID_PATTERN,
|
||||||
|
'portal-agent-run',
|
||||||
|
),
|
||||||
|
dataPolicy: 'control-plane-only-v1',
|
||||||
|
observedAt: Number.isFinite(observedAt) && observedAt > 0 ? observedAt : Date.now(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function stableRolloutBucket(value) {
|
export function stableRolloutBucket(value) {
|
||||||
const hash = crypto.createHash('sha256').update(String(value ?? '')).digest();
|
const hash = crypto.createHash('sha256').update(String(value ?? '')).digest();
|
||||||
return hash.readUInt32BE(0) % 100;
|
return hash.readUInt32BE(0) % 100;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
|
|||||||
import test from 'node:test';
|
import test from 'node:test';
|
||||||
import {
|
import {
|
||||||
buildRunEvent,
|
buildRunEvent,
|
||||||
|
normalizePageDataValidationObservation,
|
||||||
normalizeRunSpec,
|
normalizeRunSpec,
|
||||||
stableRolloutBucket,
|
stableRolloutBucket,
|
||||||
} from './contracts.mjs';
|
} from './contracts.mjs';
|
||||||
@@ -33,6 +34,59 @@ test('orchestrator contracts normalize run specs without exposing framework type
|
|||||||
assert.equal(stableRolloutBucket('run-1'), stableRolloutBucket('run-1'));
|
assert.equal(stableRolloutBucket('run-1'), stableRolloutBucket('run-1'));
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Page Data validation observations derive a bounded control-plane verdict', () => {
|
||||||
|
const observation = normalizePageDataValidationObservation({
|
||||||
|
idempotencyKey: 'run-1:page-data-delivery:v1',
|
||||||
|
taskType: 'page_data_dev_complex',
|
||||||
|
required: true,
|
||||||
|
checks: [
|
||||||
|
{ id: 'agent_run_completion', status: 'passed' },
|
||||||
|
{ id: 'page_data_binding', status: 'passed' },
|
||||||
|
{ id: 'page_data_storage_policy', status: 'passed' },
|
||||||
|
{ id: 'independent_review', status: 'skipped' },
|
||||||
|
],
|
||||||
|
metrics: { pageCount: 2, publicationCount: 1 },
|
||||||
|
source: 'portal-agent-run',
|
||||||
|
observedAt: 123,
|
||||||
|
ignoredSensitivePayload: {
|
||||||
|
workspacePath: '/private/workspace',
|
||||||
|
instruction: 'must not be projected',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
assert.equal(observation.version, 'page-data-validation-observation-v1');
|
||||||
|
assert.equal(observation.verdict, 'passed');
|
||||||
|
assert.equal(observation.dataPolicy, 'control-plane-only-v1');
|
||||||
|
assert.deepEqual(observation.metrics, { pageCount: 2, publicationCount: 1 });
|
||||||
|
assert.equal('ignoredSensitivePayload' in observation, false);
|
||||||
|
assert.equal(JSON.stringify(observation).includes('/private/workspace'), false);
|
||||||
|
|
||||||
|
const failed = normalizePageDataValidationObservation({
|
||||||
|
idempotencyKey: 'run-2:page-data-delivery:v1',
|
||||||
|
required: true,
|
||||||
|
checks: [
|
||||||
|
{
|
||||||
|
id: 'page_data_storage_policy',
|
||||||
|
status: 'failed',
|
||||||
|
codes: ['browser_storage_forbidden'],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assert.equal(failed.verdict, 'failed');
|
||||||
|
assert.deepEqual(failed.checks[0].codes, ['BROWSER_STORAGE_FORBIDDEN']);
|
||||||
|
|
||||||
|
const inconclusive = normalizePageDataValidationObservation({
|
||||||
|
idempotencyKey: 'run-3:page-data-delivery:v1',
|
||||||
|
required: true,
|
||||||
|
checks: [{ id: 'agent_run_completion', status: 'passed' }],
|
||||||
|
});
|
||||||
|
assert.equal(inconclusive.verdict, 'inconclusive');
|
||||||
|
assert.throws(
|
||||||
|
() => normalizePageDataValidationObservation({ idempotencyKey: 'missing-checks' }),
|
||||||
|
(error) => error.code === 'WORKFLOW_VALIDATION_INVALID' && error.status === 422,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
test('workflow engine registry enforces the framework-neutral contract', () => {
|
test('workflow engine registry enforces the framework-neutral contract', () => {
|
||||||
const engine = {
|
const engine = {
|
||||||
id: 'native',
|
id: 'native',
|
||||||
@@ -71,6 +125,10 @@ test('remote workflow engine speaks only the versioned orchestrator HTTP contrac
|
|||||||
await engine.resume('run-1', { approvalId: 'approval-1', decision: 'approve' });
|
await engine.resume('run-1', { approvalId: 'approval-1', decision: 'approve' });
|
||||||
await engine.cancel('run-1');
|
await engine.cancel('run-1');
|
||||||
await engine.getState('run-1');
|
await engine.getState('run-1');
|
||||||
|
await engine.recordValidationObservation('run-1', {
|
||||||
|
idempotencyKey: 'run-1:page-data-delivery:v1',
|
||||||
|
checks: [{ id: 'agent_run_completion', status: 'passed' }],
|
||||||
|
});
|
||||||
const events = [];
|
const events = [];
|
||||||
for await (const event of engine.streamEvents('run-1')) events.push(event);
|
for await (const event of engine.streamEvents('run-1')) events.push(event);
|
||||||
await engine.getExecutorJob('run-1:executor-preview');
|
await engine.getExecutorJob('run-1:executor-preview');
|
||||||
@@ -86,6 +144,10 @@ test('remote workflow engine speaks only the versioned orchestrator HTTP contrac
|
|||||||
['POST', 'http://orchestrator.internal/v1/runs/run-1/resume'],
|
['POST', 'http://orchestrator.internal/v1/runs/run-1/resume'],
|
||||||
['POST', 'http://orchestrator.internal/v1/runs/run-1/cancel'],
|
['POST', 'http://orchestrator.internal/v1/runs/run-1/cancel'],
|
||||||
['GET', 'http://orchestrator.internal/v1/runs/run-1'],
|
['GET', 'http://orchestrator.internal/v1/runs/run-1'],
|
||||||
|
[
|
||||||
|
'POST',
|
||||||
|
'http://orchestrator.internal/v1/runs/run-1/validation-observations',
|
||||||
|
],
|
||||||
['GET', 'http://orchestrator.internal/v1/runs/run-1/events'],
|
['GET', 'http://orchestrator.internal/v1/runs/run-1/events'],
|
||||||
['GET', 'http://orchestrator.internal/v1/executor-jobs/run-1%3Aexecutor-preview'],
|
['GET', 'http://orchestrator.internal/v1/executor-jobs/run-1%3Aexecutor-preview'],
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -124,6 +124,12 @@ export function createRemoteWorkflowEngine({
|
|||||||
async getState(runId) {
|
async getState(runId) {
|
||||||
return request(`/v1/runs/${encodeURIComponent(runId)}`);
|
return request(`/v1/runs/${encodeURIComponent(runId)}`);
|
||||||
},
|
},
|
||||||
|
async recordValidationObservation(runId, observation) {
|
||||||
|
return request(`/v1/runs/${encodeURIComponent(runId)}/validation-observations`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(observation ?? {}),
|
||||||
|
});
|
||||||
|
},
|
||||||
async *streamEvents(runId, cursor = null) {
|
async *streamEvents(runId, cursor = null) {
|
||||||
const query = cursor == null ? '' : `?after=${encodeURIComponent(cursor)}`;
|
const query = cursor == null ? '' : `?after=${encodeURIComponent(cursor)}`;
|
||||||
const result = await request(`/v1/runs/${encodeURIComponent(runId)}/events${query}`);
|
const result = await request(`/v1/runs/${encodeURIComponent(runId)}/events${query}`);
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
import { buildRunEvent, normalizeRunSpec } from './contracts.mjs';
|
import {
|
||||||
|
buildRunEvent,
|
||||||
|
normalizePageDataValidationObservation,
|
||||||
|
normalizeRunSpec,
|
||||||
|
} from './contracts.mjs';
|
||||||
import {
|
import {
|
||||||
CODE_RUN_SHADOW_WORKFLOW,
|
CODE_RUN_SHADOW_WORKFLOW,
|
||||||
createCodeRunShadowGraph,
|
createCodeRunShadowGraph,
|
||||||
@@ -22,6 +26,12 @@ function graphConfig(runId) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function validationObservationFingerprint(observation) {
|
||||||
|
if (!observation || typeof observation !== 'object') return '';
|
||||||
|
const { observedAt: _observedAt, ...stable } = observation;
|
||||||
|
return JSON.stringify(stable);
|
||||||
|
}
|
||||||
|
|
||||||
function projectSnapshot(runId, snapshot) {
|
function projectSnapshot(runId, snapshot) {
|
||||||
const values = snapshot?.values ?? {};
|
const values = snapshot?.values ?? {};
|
||||||
if (!values.spec) return null;
|
if (!values.spec) return null;
|
||||||
@@ -34,6 +44,7 @@ function projectSnapshot(runId, snapshot) {
|
|||||||
phase: values.phase ?? null,
|
phase: values.phase ?? null,
|
||||||
plan: values.plan ?? null,
|
plan: values.plan ?? null,
|
||||||
result: values.result ?? null,
|
result: values.result ?? null,
|
||||||
|
validation: values.validation ?? null,
|
||||||
createdAt: values.events?.[0]?.timestamp ?? null,
|
createdAt: values.events?.[0]?.timestamp ?? null,
|
||||||
updatedAt: values.events?.at?.(-1)?.timestamp ?? null,
|
updatedAt: values.events?.at?.(-1)?.timestamp ?? null,
|
||||||
};
|
};
|
||||||
@@ -136,6 +147,49 @@ export function createLangGraphOrchestratorRuntime({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function recordValidationObservation(runId, input = {}) {
|
||||||
|
const normalizedRunId = String(runId ?? '').trim();
|
||||||
|
if (!normalizedRunId) return null;
|
||||||
|
const snapshot = await getSnapshot(normalizedRunId);
|
||||||
|
const values = snapshot?.values ?? {};
|
||||||
|
if (!values.spec) return null;
|
||||||
|
const projected = projectSnapshot(normalizedRunId, snapshot);
|
||||||
|
if (!TERMINAL_RUN_STATUSES.has(projected.status)) {
|
||||||
|
const error = new Error('Workflow run must be terminal before validation observation');
|
||||||
|
error.code = 'WORKFLOW_VALIDATION_RUN_NOT_TERMINAL';
|
||||||
|
error.status = 409;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
const observation = normalizePageDataValidationObservation(input);
|
||||||
|
const existing = values.validation ?? null;
|
||||||
|
if (existing?.idempotencyKey === observation.idempotencyKey) {
|
||||||
|
if (
|
||||||
|
validationObservationFingerprint(existing)
|
||||||
|
!== validationObservationFingerprint(observation)
|
||||||
|
) {
|
||||||
|
const error = new Error('Validation observation idempotency key conflicts with existing data');
|
||||||
|
error.code = 'WORKFLOW_VALIDATION_IDEMPOTENCY_CONFLICT';
|
||||||
|
error.status = 409;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
return projected;
|
||||||
|
}
|
||||||
|
await graph.updateState(
|
||||||
|
graphConfig(normalizedRunId),
|
||||||
|
{
|
||||||
|
validation: observation,
|
||||||
|
events: [buildRunEvent({
|
||||||
|
runId: normalizedRunId,
|
||||||
|
sequence: (values.events?.length ?? 0) + 1,
|
||||||
|
type: 'workflow_validation_observed',
|
||||||
|
data: observation,
|
||||||
|
})],
|
||||||
|
},
|
||||||
|
'finalize_run',
|
||||||
|
);
|
||||||
|
return getState(normalizedRunId);
|
||||||
|
}
|
||||||
|
|
||||||
async function deleteRun(runId) {
|
async function deleteRun(runId) {
|
||||||
const normalizedRunId = String(runId ?? '').trim();
|
const normalizedRunId = String(runId ?? '').trim();
|
||||||
if (!normalizedRunId) return null;
|
if (!normalizedRunId) return null;
|
||||||
@@ -204,6 +258,7 @@ export function createLangGraphOrchestratorRuntime({
|
|||||||
},
|
},
|
||||||
|
|
||||||
getState,
|
getState,
|
||||||
|
recordValidationObservation,
|
||||||
|
|
||||||
async listEvents(runId, { after = 0 } = {}) {
|
async listEvents(runId, { after = 0 } = {}) {
|
||||||
const normalizedRunId = String(runId ?? '').trim();
|
const normalizedRunId = String(runId ?? '').trim();
|
||||||
@@ -397,4 +452,5 @@ export const orchestratorRuntimeInternals = {
|
|||||||
graphConfig,
|
graphConfig,
|
||||||
projectSnapshot,
|
projectSnapshot,
|
||||||
TERMINAL_RUN_STATUSES,
|
TERMINAL_RUN_STATUSES,
|
||||||
|
validationObservationFingerprint,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -87,6 +87,61 @@ test('LangGraph runtime is idempotent by run id', async () => {
|
|||||||
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 3);
|
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 3);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('LangGraph runtime records idempotent Page Data validation evidence after Shadow completion', async () => {
|
||||||
|
const runtime = createLangGraphOrchestratorRuntime({
|
||||||
|
checkpointer: new MemorySaver(),
|
||||||
|
});
|
||||||
|
await runtime.start(runSpec({
|
||||||
|
input: {
|
||||||
|
instruction: '[shadow-control-plane-only]',
|
||||||
|
taskType: 'page_data_dev_complex',
|
||||||
|
pageDataRequired: true,
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
const input = {
|
||||||
|
idempotencyKey: 'run-shadow-1:page-data-delivery:v1',
|
||||||
|
taskType: 'page_data_dev_complex',
|
||||||
|
required: true,
|
||||||
|
checks: [
|
||||||
|
{ id: 'agent_run_completion', status: 'passed' },
|
||||||
|
{ id: 'page_data_binding', status: 'passed' },
|
||||||
|
{ id: 'page_data_storage_policy', status: 'passed' },
|
||||||
|
],
|
||||||
|
metrics: { pageCount: 2, publicationCount: 1 },
|
||||||
|
observedAt: 123,
|
||||||
|
};
|
||||||
|
|
||||||
|
const recorded = await runtime.recordValidationObservation('run-shadow-1', input);
|
||||||
|
assert.equal(recorded.status, 'succeeded');
|
||||||
|
assert.equal(recorded.validation.verdict, 'passed');
|
||||||
|
assert.equal(recorded.validation.dataPolicy, 'control-plane-only-v1');
|
||||||
|
assert.deepEqual(
|
||||||
|
(await runtime.listEvents('run-shadow-1')).events.map((event) => event.type),
|
||||||
|
[
|
||||||
|
'workflow_validated',
|
||||||
|
'workflow_planned',
|
||||||
|
'workflow_completed',
|
||||||
|
'workflow_validation_observed',
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
|
const duplicate = await runtime.recordValidationObservation('run-shadow-1', {
|
||||||
|
...input,
|
||||||
|
observedAt: 456,
|
||||||
|
});
|
||||||
|
assert.deepEqual(duplicate, recorded);
|
||||||
|
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 4);
|
||||||
|
await assert.rejects(
|
||||||
|
() => runtime.recordValidationObservation('run-shadow-1', {
|
||||||
|
...input,
|
||||||
|
checks: [{ id: 'page_data_binding', status: 'failed' }],
|
||||||
|
}),
|
||||||
|
(error) => error.code === 'WORKFLOW_VALIDATION_IDEMPOTENCY_CONFLICT'
|
||||||
|
&& error.status === 409,
|
||||||
|
);
|
||||||
|
assert.equal(await runtime.recordValidationObservation('missing-run', input), null);
|
||||||
|
});
|
||||||
|
|
||||||
test('LangGraph runtime deletes terminal checkpoints and linked Executor Jobs', async () => {
|
test('LangGraph runtime deletes terminal checkpoints and linked Executor Jobs', async () => {
|
||||||
const runtime = createLangGraphOrchestratorRuntime({
|
const runtime = createLangGraphOrchestratorRuntime({
|
||||||
checkpointer: new MemorySaver(),
|
checkpointer: new MemorySaver(),
|
||||||
|
|||||||
@@ -1,9 +1,12 @@
|
|||||||
import {
|
import {
|
||||||
WORKFLOW_ENGINE,
|
WORKFLOW_ENGINE,
|
||||||
|
normalizePageDataValidationObservation,
|
||||||
normalizeRunSpec,
|
normalizeRunSpec,
|
||||||
} from './contracts.mjs';
|
} from './contracts.mjs';
|
||||||
import { createRemoteWorkflowEngine } from './engine-registry.mjs';
|
import { createRemoteWorkflowEngine } from './engine-registry.mjs';
|
||||||
|
|
||||||
|
const VALIDATION_RETRY_DELAYS_MS = Object.freeze([25, 75, 225]);
|
||||||
|
|
||||||
function safeError(error) {
|
function safeError(error) {
|
||||||
return {
|
return {
|
||||||
code: String(error?.code ?? 'WORKFLOW_SHADOW_FAILED').slice(0, 128),
|
code: String(error?.code ?? 'WORKFLOW_SHADOW_FAILED').slice(0, 128),
|
||||||
@@ -11,6 +14,20 @@ function safeError(error) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function recordValidationWithRetry(engine, runId, observation) {
|
||||||
|
let attempt = 0;
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
return await engine.recordValidationObservation(runId, observation);
|
||||||
|
} catch (error) {
|
||||||
|
const delayMs = VALIDATION_RETRY_DELAYS_MS[attempt];
|
||||||
|
if (error?.status !== 404 || delayMs == null) throw error;
|
||||||
|
attempt += 1;
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function createWorkflowShadowObserver({
|
export function createWorkflowShadowObserver({
|
||||||
configService,
|
configService,
|
||||||
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
|
||||||
@@ -21,14 +38,43 @@ export function createWorkflowShadowObserver({
|
|||||||
throw new Error('Workflow shadow observer requires orchestrator config service');
|
throw new Error('Workflow shadow observer requires orchestrator config service');
|
||||||
}
|
}
|
||||||
|
|
||||||
return async function observeWorkflowRun({
|
async function selectShadowEngine({
|
||||||
|
runId,
|
||||||
|
requestId,
|
||||||
|
userId,
|
||||||
|
workflowName,
|
||||||
|
}) {
|
||||||
|
const selection = await configService.selectEngine({
|
||||||
|
runId,
|
||||||
|
requestId,
|
||||||
|
userId,
|
||||||
|
workflowName,
|
||||||
|
});
|
||||||
|
if (selection.shadowEngine !== WORKFLOW_ENGINE.LANGGRAPH) {
|
||||||
|
return { selection, engine: null };
|
||||||
|
}
|
||||||
|
const state = await configService.getRuntimeState();
|
||||||
|
return {
|
||||||
|
selection,
|
||||||
|
engine: createRemoteWorkflowEngine({
|
||||||
|
id: WORKFLOW_ENGINE.LANGGRAPH,
|
||||||
|
baseUrl: state.config.serviceUrl,
|
||||||
|
serviceToken,
|
||||||
|
timeoutMs: state.config.requestTimeoutMs,
|
||||||
|
fetchImpl,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function observeWorkflowRun({
|
||||||
runId,
|
runId,
|
||||||
requestId,
|
requestId,
|
||||||
userId,
|
userId,
|
||||||
workflowName = 'code-run-v1',
|
workflowName = 'code-run-v1',
|
||||||
taskType = null,
|
taskType = null,
|
||||||
|
pageDataRequired = false,
|
||||||
} = {}) {
|
} = {}) {
|
||||||
const selection = await configService.selectEngine({
|
const { selection, engine } = await selectShadowEngine({
|
||||||
runId,
|
runId,
|
||||||
requestId,
|
requestId,
|
||||||
userId,
|
userId,
|
||||||
@@ -49,7 +95,7 @@ export function createWorkflowShadowObserver({
|
|||||||
dryRun: true,
|
dryRun: true,
|
||||||
handoffAllowed: false,
|
handoffAllowed: false,
|
||||||
} : null;
|
} : null;
|
||||||
if (selection.shadowEngine !== WORKFLOW_ENGINE.LANGGRAPH) {
|
if (!engine) {
|
||||||
return {
|
return {
|
||||||
observed: false,
|
observed: false,
|
||||||
reason: selection.reason,
|
reason: selection.reason,
|
||||||
@@ -58,14 +104,6 @@ export function createWorkflowShadowObserver({
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const state = await configService.getRuntimeState();
|
|
||||||
const engine = createRemoteWorkflowEngine({
|
|
||||||
id: WORKFLOW_ENGINE.LANGGRAPH,
|
|
||||||
baseUrl: state.config.serviceUrl,
|
|
||||||
serviceToken,
|
|
||||||
timeoutMs: state.config.requestTimeoutMs,
|
|
||||||
fetchImpl,
|
|
||||||
});
|
|
||||||
const spec = normalizeRunSpec({
|
const spec = normalizeRunSpec({
|
||||||
runId,
|
runId,
|
||||||
requestId,
|
requestId,
|
||||||
@@ -75,6 +113,7 @@ export function createWorkflowShadowObserver({
|
|||||||
instruction: '[shadow-control-plane-only]',
|
instruction: '[shadow-control-plane-only]',
|
||||||
taskType,
|
taskType,
|
||||||
toolMode: 'code',
|
toolMode: 'code',
|
||||||
|
pageDataRequired: pageDataRequired === true,
|
||||||
},
|
},
|
||||||
policy: {
|
policy: {
|
||||||
executionMode: 'observe-only',
|
executionMode: 'observe-only',
|
||||||
@@ -104,9 +143,49 @@ export function createWorkflowShadowObserver({
|
|||||||
}
|
}
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
observeWorkflowRun.observeValidation = async function observeWorkflowValidation({
|
||||||
|
runId,
|
||||||
|
requestId,
|
||||||
|
userId,
|
||||||
|
workflowName = 'code-run-v1',
|
||||||
|
observation,
|
||||||
|
} = {}) {
|
||||||
|
const { selection, engine } = await selectShadowEngine({
|
||||||
|
runId,
|
||||||
|
requestId,
|
||||||
|
userId,
|
||||||
|
workflowName,
|
||||||
|
});
|
||||||
|
if (!engine) {
|
||||||
|
return {
|
||||||
|
observed: false,
|
||||||
|
reason: selection.reason,
|
||||||
|
mode: selection.mode,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const normalized = normalizePageDataValidationObservation(observation);
|
||||||
|
try {
|
||||||
|
const result = await recordValidationWithRetry(engine, runId, normalized);
|
||||||
|
return {
|
||||||
|
observed: true,
|
||||||
|
engine: WORKFLOW_ENGINE.LANGGRAPH,
|
||||||
|
mode: selection.mode,
|
||||||
|
configVersion: selection.configVersion,
|
||||||
|
validation: result.validation ?? normalized,
|
||||||
|
shadowRun: result,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
logger.warn('[orchestrator-shadow] validation observation failed:', safeError(error));
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
return observeWorkflowRun;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const workflowShadowObserverInternals = {
|
export const workflowShadowObserverInternals = {
|
||||||
|
recordValidationWithRetry,
|
||||||
safeError,
|
safeError,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -179,6 +179,7 @@ test('shadow observer sends a control-plane-only RunSpec without user content or
|
|||||||
userId: 'user-2',
|
userId: 'user-2',
|
||||||
sessionId: 'session-2',
|
sessionId: 'session-2',
|
||||||
taskType: 'code-change',
|
taskType: 'code-change',
|
||||||
|
pageDataRequired: true,
|
||||||
userMessage: {
|
userMessage: {
|
||||||
content: [{ type: 'text', text: 'Implement the service boundary' }],
|
content: [{ type: 'text', text: 'Implement the service boundary' }],
|
||||||
},
|
},
|
||||||
@@ -192,6 +193,7 @@ test('shadow observer sends a control-plane-only RunSpec without user content or
|
|||||||
assert.equal(body.policy.executionMode, 'observe-only');
|
assert.equal(body.policy.executionMode, 'observe-only');
|
||||||
assert.equal(body.policy.sideEffectsAllowed, false);
|
assert.equal(body.policy.sideEffectsAllowed, false);
|
||||||
assert.equal(body.input.instruction, '[shadow-control-plane-only]');
|
assert.equal(body.input.instruction, '[shadow-control-plane-only]');
|
||||||
|
assert.equal(body.input.pageDataRequired, true);
|
||||||
assert.equal('sessionRef' in body.input, false);
|
assert.equal('sessionRef' in body.input, false);
|
||||||
assert.deepEqual(body.subject, { tenantId: null, userId: null });
|
assert.deepEqual(body.subject, { tenantId: null, userId: null });
|
||||||
assert.equal(body.metadata.configVersion, 7);
|
assert.equal(body.metadata.configVersion, 7);
|
||||||
@@ -270,6 +272,37 @@ test('isolated Portal shadow flow reaches LangGraph over HTTP and removes all te
|
|||||||
assert.equal(checkpointJson.includes('private-user-id'), false);
|
assert.equal(checkpointJson.includes('private-user-id'), false);
|
||||||
assert.equal(checkpointJson.includes('private-session-id'), false);
|
assert.equal(checkpointJson.includes('private-session-id'), false);
|
||||||
|
|
||||||
|
const validation = await observer.observeValidation({
|
||||||
|
runId: 'isolated-shadow-run-1',
|
||||||
|
requestId: 'isolated-shadow-request-1',
|
||||||
|
userId: 'private-user-id',
|
||||||
|
workflowName: 'code-run-v1',
|
||||||
|
observation: {
|
||||||
|
idempotencyKey: 'isolated-shadow-run-1:page-data-delivery:v1',
|
||||||
|
taskType: 'page_data_dev_complex',
|
||||||
|
required: true,
|
||||||
|
checks: [
|
||||||
|
{ id: 'agent_run_completion', status: 'passed' },
|
||||||
|
{ id: 'page_data_binding', status: 'passed' },
|
||||||
|
{ id: 'page_data_storage_policy', status: 'passed' },
|
||||||
|
],
|
||||||
|
metrics: { pageCount: 2, publicationCount: 1 },
|
||||||
|
source: 'portal-agent-run',
|
||||||
|
observedAt: 123,
|
||||||
|
sensitive: {
|
||||||
|
workspacePath: '/private/workspace',
|
||||||
|
instruction: secretUserContent,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(validation.observed, true);
|
||||||
|
assert.equal(validation.validation.verdict, 'passed');
|
||||||
|
const validatedState = await runtime.getState('isolated-shadow-run-1');
|
||||||
|
assert.equal(validatedState.validation.verdict, 'passed');
|
||||||
|
const validatedJson = JSON.stringify(validatedState.validation);
|
||||||
|
assert.equal(validatedJson.includes('/private/workspace'), false);
|
||||||
|
assert.equal(validatedJson.includes(secretUserContent), false);
|
||||||
|
|
||||||
const deleted = await fetch(`${server.baseUrl}/v1/runs/isolated-shadow-run-1`, {
|
const deleted = await fetch(`${server.baseUrl}/v1/runs/isolated-shadow-run-1`, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
headers: { authorization: 'Bearer isolated-shadow-token' },
|
headers: { authorization: 'Bearer isolated-shadow-token' },
|
||||||
|
|||||||
Reference in New Issue
Block a user