tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat(orchestrator): observe Page Data validation

Browse Source
This commit is contained in:
john
2026-07-25 14:16:57 +08:00
parent e02495a6c0
commit 417aa9c78b
17 changed files with 784 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/orchestrator/app.mjs
+13
View File
@@ -134,6 +134,19 @@ export function createOrchestratorApp({
}
});
app.post('/v1/runs/:runId/validation-observations', async (request, response, next) => {
try {
const state = await runtime.recordValidationObservation(
request.params.runId,
request.body,
);
if (!state) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
return response.json(state);
} catch (error) {
return next(error);
}
});
app.post('/v1/runs/:runId/resume', async (request, response, next) => {
try {
const state = await runtime.resume(request.params.runId, request.body);
services/orchestrator/app.test.mjs
+27 -1
View File
@@ -76,12 +76,38 @@ test('orchestrator HTTP API exposes health and authenticated run endpoints', asy
assert.equal(created.status, 202);
assert.equal((await created.json()).status, 'succeeded');
const validation = await fetch(
`${server.baseUrl}/v1/runs/http-shadow-1/validation-observations`,
{
method: 'POST',
headers: {
authorization: 'Bearer test-service-token',
'content-type': 'application/json',
},
body: JSON.stringify({
idempotencyKey: 'http-shadow-1:page-data-delivery:v1',
taskType: 'page_data_dev',
required: true,
checks: [
{ id: 'agent_run_completion', status: 'passed' },
{ id: 'page_data_binding', status: 'passed' },
{ id: 'page_data_storage_policy', status: 'passed' },
],
metrics: { pageCount: 1, publicationCount: 1 },
observedAt: 123,
}),
},
);
assert.equal(validation.status, 200);
assert.equal((await validation.json()).validation.verdict, 'passed');
const events = await fetch(`${server.baseUrl}/v1/runs/http-shadow-1/events?after=1`, {
headers: { authorization: 'Bearer test-service-token' },
});
const eventBody = await events.json();
assert.equal(events.status, 200);
assert.deepEqual(eventBody.events.map((event) => event.sequence), [2, 3]);
assert.deepEqual(eventBody.events.map((event) => event.sequence), [2, 3, 4]);
assert.equal(eventBody.events.at(-1).type, 'workflow_validation_observed');
const jobId = 'http-shadow-1:executor-preview';
const executorJob = await fetch(
services/orchestrator/code-run-shadow-graph.mjs
+4
View File
@@ -30,6 +30,10 @@ const ShadowState = Annotation.Root({
reducer: (_current, update) => update,
default: () => null,
}),
validation: Annotation({
reducer: (_current, update) => update,
default: () => null,
}),
events: Annotation({
reducer: (current, update) => [...current, ...update],
default: () => [],
services/orchestrator/contracts.mjs
+86
View File
@@ -16,8 +16,18 @@ export const DEFAULT_ORCHESTRATED_WORKFLOWS = Object.freeze([
'code-run-v1',
]);
export const PAGE_DATA_VALIDATION_OBSERVATION_VERSION =
'page-data-validation-observation-v1';
const ENGINE_ID_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/;
const WORKFLOW_NAME_PATTERN = /^[a-z][a-z0-9_-]{0,127}$/;
const VALIDATION_CHECK_ID_PATTERN = /^[a-z][a-z0-9_-]{0,127}$/;
const VALIDATION_CHECK_STATUSES = new Set(['passed', 'failed', 'skipped']);
const REQUIRED_PAGE_DATA_CHECKS = Object.freeze([
'agent_run_completion',
'page_data_binding',
'page_data_storage_policy',
]);
function normalizeIdentifier(value, pattern, fallback) {
const normalized = String(value ?? '').trim().toLowerCase();
@@ -98,6 +108,82 @@ export function buildRunEvent({
};
}
function validationContractError(message) {
const error = new Error(message);
error.code = 'WORKFLOW_VALIDATION_INVALID';
error.status = 422;
return error;
}
function normalizeValidationCount(value) {
const parsed = Number(value);
if (!Number.isFinite(parsed) || parsed < 0) return 0;
return Math.min(1_000_000, Math.floor(parsed));
}
export function normalizePageDataValidationObservation(input = {}) {
const source = input && typeof input === 'object' && !Array.isArray(input) ? input : {};
const idempotencyKey = String(source.idempotencyKey ?? '').trim().slice(0, 200);
if (!idempotencyKey) {
throw validationContractError('Page Data validation observation requires idempotencyKey');
}
if (!Array.isArray(source.checks) || source.checks.length === 0) {
throw validationContractError('Page Data validation observation requires checks');
}
const checks = source.checks.slice(0, 32).map((item) => {
const check = item && typeof item === 'object' && !Array.isArray(item) ? item : {};
const id = normalizeIdentifier(check.id, VALIDATION_CHECK_ID_PATTERN, '');
const status = String(check.status ?? '').trim().toLowerCase();
if (!id || !VALIDATION_CHECK_STATUSES.has(status)) {
throw validationContractError('Page Data validation observation contains an invalid check');
}
return {
id,
status,
codes: normalizeStringList(check.codes, {
limit: 32,
itemLimit: 128,
normalize: (value) => String(value ?? '').trim().toUpperCase(),
}),
};
});
const byId = new Map(checks.map((check) => [check.id, check]));
const failed = checks.some((check) => check.status === 'failed');
const required = source.required === true;
const requiredChecksPassed = REQUIRED_PAGE_DATA_CHECKS.every(
(id) => byId.get(id)?.status === 'passed',
);
const verdict = failed
? 'failed'
: required && !requiredChecksPassed
? 'inconclusive'
: checks.some((check) => check.status === 'passed')
? 'passed'
: 'inconclusive';
const taskType = normalizeWorkflowName(source.taskType, '');
const observedAt = Number(source.observedAt);
return {
version: PAGE_DATA_VALIDATION_OBSERVATION_VERSION,
kind: 'page-data-delivery',
idempotencyKey,
taskType: taskType || null,
required,
verdict,
checks,
metrics: {
pageCount: normalizeValidationCount(source.metrics?.pageCount),
publicationCount: normalizeValidationCount(source.metrics?.publicationCount),
},
source: normalizeIdentifier(
source.source,
VALIDATION_CHECK_ID_PATTERN,
'portal-agent-run',
),
dataPolicy: 'control-plane-only-v1',
observedAt: Number.isFinite(observedAt) && observedAt > 0 ? observedAt : Date.now(),
};
}
export function stableRolloutBucket(value) {
const hash = crypto.createHash('sha256').update(String(value ?? '')).digest();
return hash.readUInt32BE(0) % 100;
services/orchestrator/contracts.test.mjs
+62
View File
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
import test from 'node:test';
import {
buildRunEvent,
normalizePageDataValidationObservation,
normalizeRunSpec,
stableRolloutBucket,
} from './contracts.mjs';
@@ -33,6 +34,59 @@ test('orchestrator contracts normalize run specs without exposing framework type
assert.equal(stableRolloutBucket('run-1'), stableRolloutBucket('run-1'));
});
test('Page Data validation observations derive a bounded control-plane verdict', () => {
const observation = normalizePageDataValidationObservation({
idempotencyKey: 'run-1:page-data-delivery:v1',
taskType: 'page_data_dev_complex',
required: true,
checks: [
{ id: 'agent_run_completion', status: 'passed' },
{ id: 'page_data_binding', status: 'passed' },
{ id: 'page_data_storage_policy', status: 'passed' },
{ id: 'independent_review', status: 'skipped' },
],
metrics: { pageCount: 2, publicationCount: 1 },
source: 'portal-agent-run',
observedAt: 123,
ignoredSensitivePayload: {
workspacePath: '/private/workspace',
instruction: 'must not be projected',
},
});
assert.equal(observation.version, 'page-data-validation-observation-v1');
assert.equal(observation.verdict, 'passed');
assert.equal(observation.dataPolicy, 'control-plane-only-v1');
assert.deepEqual(observation.metrics, { pageCount: 2, publicationCount: 1 });
assert.equal('ignoredSensitivePayload' in observation, false);
assert.equal(JSON.stringify(observation).includes('/private/workspace'), false);
const failed = normalizePageDataValidationObservation({
idempotencyKey: 'run-2:page-data-delivery:v1',
required: true,
checks: [
{
id: 'page_data_storage_policy',
status: 'failed',
codes: ['browser_storage_forbidden'],
},
],
});
assert.equal(failed.verdict, 'failed');
assert.deepEqual(failed.checks[0].codes, ['BROWSER_STORAGE_FORBIDDEN']);
const inconclusive = normalizePageDataValidationObservation({
idempotencyKey: 'run-3:page-data-delivery:v1',
required: true,
checks: [{ id: 'agent_run_completion', status: 'passed' }],
});
assert.equal(inconclusive.verdict, 'inconclusive');
assert.throws(
() => normalizePageDataValidationObservation({ idempotencyKey: 'missing-checks' }),
(error) => error.code === 'WORKFLOW_VALIDATION_INVALID' && error.status === 422,
);
});
test('workflow engine registry enforces the framework-neutral contract', () => {
const engine = {
id: 'native',
@@ -71,6 +125,10 @@ test('remote workflow engine speaks only the versioned orchestrator HTTP contrac
await engine.resume('run-1', { approvalId: 'approval-1', decision: 'approve' });
await engine.cancel('run-1');
await engine.getState('run-1');
await engine.recordValidationObservation('run-1', {
idempotencyKey: 'run-1:page-data-delivery:v1',
checks: [{ id: 'agent_run_completion', status: 'passed' }],
});
const events = [];
for await (const event of engine.streamEvents('run-1')) events.push(event);
await engine.getExecutorJob('run-1:executor-preview');
@@ -86,6 +144,10 @@ test('remote workflow engine speaks only the versioned orchestrator HTTP contrac
['POST', 'http://orchestrator.internal/v1/runs/run-1/resume'],
['POST', 'http://orchestrator.internal/v1/runs/run-1/cancel'],
['GET', 'http://orchestrator.internal/v1/runs/run-1'],
[
'POST',
'http://orchestrator.internal/v1/runs/run-1/validation-observations',
],
['GET', 'http://orchestrator.internal/v1/runs/run-1/events'],
['GET', 'http://orchestrator.internal/v1/executor-jobs/run-1%3Aexecutor-preview'],
[
services/orchestrator/engine-registry.mjs
+6
View File
@@ -124,6 +124,12 @@ export function createRemoteWorkflowEngine({
async getState(runId) {
return request(`/v1/runs/${encodeURIComponent(runId)}`);
},
async recordValidationObservation(runId, observation) {
return request(`/v1/runs/${encodeURIComponent(runId)}/validation-observations`, {
method: 'POST',
body: JSON.stringify(observation ?? {}),
});
},
async *streamEvents(runId, cursor = null) {
const query = cursor == null ? '' : `?after=${encodeURIComponent(cursor)}`;
const result = await request(`/v1/runs/${encodeURIComponent(runId)}/events${query}`);
services/orchestrator/runtime.mjs
+57 -1
View File
@@ -1,4 +1,8 @@
import { buildRunEvent, normalizeRunSpec } from './contracts.mjs';
import {
buildRunEvent,
normalizePageDataValidationObservation,
normalizeRunSpec,
} from './contracts.mjs';
import {
CODE_RUN_SHADOW_WORKFLOW,
createCodeRunShadowGraph,
@@ -22,6 +26,12 @@ function graphConfig(runId) {
};
}
function validationObservationFingerprint(observation) {
if (!observation || typeof observation !== 'object') return '';
const { observedAt: _observedAt, ...stable } = observation;
return JSON.stringify(stable);
}
function projectSnapshot(runId, snapshot) {
const values = snapshot?.values ?? {};
if (!values.spec) return null;
@@ -34,6 +44,7 @@ function projectSnapshot(runId, snapshot) {
phase: values.phase ?? null,
plan: values.plan ?? null,
result: values.result ?? null,
validation: values.validation ?? null,
createdAt: values.events?.[0]?.timestamp ?? null,
updatedAt: values.events?.at?.(-1)?.timestamp ?? null,
};
@@ -136,6 +147,49 @@ export function createLangGraphOrchestratorRuntime({
);
}
async function recordValidationObservation(runId, input = {}) {
const normalizedRunId = String(runId ?? '').trim();
if (!normalizedRunId) return null;
const snapshot = await getSnapshot(normalizedRunId);
const values = snapshot?.values ?? {};
if (!values.spec) return null;
const projected = projectSnapshot(normalizedRunId, snapshot);
if (!TERMINAL_RUN_STATUSES.has(projected.status)) {
const error = new Error('Workflow run must be terminal before validation observation');
error.code = 'WORKFLOW_VALIDATION_RUN_NOT_TERMINAL';
error.status = 409;
throw error;
}
const observation = normalizePageDataValidationObservation(input);
const existing = values.validation ?? null;
if (existing?.idempotencyKey === observation.idempotencyKey) {
if (
validationObservationFingerprint(existing)
!== validationObservationFingerprint(observation)
) {
const error = new Error('Validation observation idempotency key conflicts with existing data');
error.code = 'WORKFLOW_VALIDATION_IDEMPOTENCY_CONFLICT';
error.status = 409;
throw error;
}
return projected;
}
await graph.updateState(
graphConfig(normalizedRunId),
{
validation: observation,
events: [buildRunEvent({
runId: normalizedRunId,
sequence: (values.events?.length ?? 0) + 1,
type: 'workflow_validation_observed',
data: observation,
})],
},
'finalize_run',
);
return getState(normalizedRunId);
}
async function deleteRun(runId) {
const normalizedRunId = String(runId ?? '').trim();
if (!normalizedRunId) return null;
@@ -204,6 +258,7 @@ export function createLangGraphOrchestratorRuntime({
},
getState,
recordValidationObservation,
async listEvents(runId, { after = 0 } = {}) {
const normalizedRunId = String(runId ?? '').trim();
@@ -397,4 +452,5 @@ export const orchestratorRuntimeInternals = {
graphConfig,
projectSnapshot,
TERMINAL_RUN_STATUSES,
validationObservationFingerprint,
};
services/orchestrator/runtime.test.mjs
+55
View File
@@ -87,6 +87,61 @@ test('LangGraph runtime is idempotent by run id', async () => {
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 3);
});
test('LangGraph runtime records idempotent Page Data validation evidence after Shadow completion', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
await runtime.start(runSpec({
input: {
instruction: '[shadow-control-plane-only]',
taskType: 'page_data_dev_complex',
pageDataRequired: true,
},
}));
const input = {
idempotencyKey: 'run-shadow-1:page-data-delivery:v1',
taskType: 'page_data_dev_complex',
required: true,
checks: [
{ id: 'agent_run_completion', status: 'passed' },
{ id: 'page_data_binding', status: 'passed' },
{ id: 'page_data_storage_policy', status: 'passed' },
],
metrics: { pageCount: 2, publicationCount: 1 },
observedAt: 123,
};
const recorded = await runtime.recordValidationObservation('run-shadow-1', input);
assert.equal(recorded.status, 'succeeded');
assert.equal(recorded.validation.verdict, 'passed');
assert.equal(recorded.validation.dataPolicy, 'control-plane-only-v1');
assert.deepEqual(
(await runtime.listEvents('run-shadow-1')).events.map((event) => event.type),
[
'workflow_validated',
'workflow_planned',
'workflow_completed',
'workflow_validation_observed',
],
);
const duplicate = await runtime.recordValidationObservation('run-shadow-1', {
...input,
observedAt: 456,
});
assert.deepEqual(duplicate, recorded);
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 4);
await assert.rejects(
() => runtime.recordValidationObservation('run-shadow-1', {
...input,
checks: [{ id: 'page_data_binding', status: 'failed' }],
}),
(error) => error.code === 'WORKFLOW_VALIDATION_IDEMPOTENCY_CONFLICT'
&& error.status === 409,
);
assert.equal(await runtime.recordValidationObservation('missing-run', input), null);
});
test('LangGraph runtime deletes terminal checkpoints and linked Executor Jobs', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
services/orchestrator/shadow-observer.mjs
+90 -11
View File
@@ -1,9 +1,12 @@
import {
WORKFLOW_ENGINE,
normalizePageDataValidationObservation,
normalizeRunSpec,
} from './contracts.mjs';
import { createRemoteWorkflowEngine } from './engine-registry.mjs';
const VALIDATION_RETRY_DELAYS_MS = Object.freeze([25, 75, 225]);
function safeError(error) {
return {
code: String(error?.code ?? 'WORKFLOW_SHADOW_FAILED').slice(0, 128),
@@ -11,6 +14,20 @@ function safeError(error) {
};
}
async function recordValidationWithRetry(engine, runId, observation) {
let attempt = 0;
while (true) {
try {
return await engine.recordValidationObservation(runId, observation);
} catch (error) {
const delayMs = VALIDATION_RETRY_DELAYS_MS[attempt];
if (error?.status !== 404 || delayMs == null) throw error;
attempt += 1;
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}
}
export function createWorkflowShadowObserver({
configService,
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
@@ -21,14 +38,43 @@ export function createWorkflowShadowObserver({
throw new Error('Workflow shadow observer requires orchestrator config service');
}
return async function observeWorkflowRun({
async function selectShadowEngine({
runId,
requestId,
userId,
workflowName,
}) {
const selection = await configService.selectEngine({
runId,
requestId,
userId,
workflowName,
});
if (selection.shadowEngine !== WORKFLOW_ENGINE.LANGGRAPH) {
return { selection, engine: null };
}
const state = await configService.getRuntimeState();
return {
selection,
engine: createRemoteWorkflowEngine({
id: WORKFLOW_ENGINE.LANGGRAPH,
baseUrl: state.config.serviceUrl,
serviceToken,
timeoutMs: state.config.requestTimeoutMs,
fetchImpl,
}),
};
}
async function observeWorkflowRun({
runId,
requestId,
userId,
workflowName = 'code-run-v1',
taskType = null,
pageDataRequired = false,
} = {}) {
const selection = await configService.selectEngine({
const { selection, engine } = await selectShadowEngine({
runId,
requestId,
userId,
@@ -49,7 +95,7 @@ export function createWorkflowShadowObserver({
dryRun: true,
handoffAllowed: false,
} : null;
if (selection.shadowEngine !== WORKFLOW_ENGINE.LANGGRAPH) {
if (!engine) {
return {
observed: false,
reason: selection.reason,
@@ -58,14 +104,6 @@ export function createWorkflowShadowObserver({
};
}
const state = await configService.getRuntimeState();
const engine = createRemoteWorkflowEngine({
id: WORKFLOW_ENGINE.LANGGRAPH,
baseUrl: state.config.serviceUrl,
serviceToken,
timeoutMs: state.config.requestTimeoutMs,
fetchImpl,
});
const spec = normalizeRunSpec({
runId,
requestId,
@@ -75,6 +113,7 @@ export function createWorkflowShadowObserver({
instruction: '[shadow-control-plane-only]',
taskType,
toolMode: 'code',
pageDataRequired: pageDataRequired === true,
},
policy: {
executionMode: 'observe-only',
@@ -104,9 +143,49 @@ export function createWorkflowShadowObserver({
}
throw error;
}
}
observeWorkflowRun.observeValidation = async function observeWorkflowValidation({
runId,
requestId,
userId,
workflowName = 'code-run-v1',
observation,
} = {}) {
const { selection, engine } = await selectShadowEngine({
runId,
requestId,
userId,
workflowName,
});
if (!engine) {
return {
observed: false,
reason: selection.reason,
mode: selection.mode,
};
}
const normalized = normalizePageDataValidationObservation(observation);
try {
const result = await recordValidationWithRetry(engine, runId, normalized);
return {
observed: true,
engine: WORKFLOW_ENGINE.LANGGRAPH,
mode: selection.mode,
configVersion: selection.configVersion,
validation: result.validation ?? normalized,
shadowRun: result,
};
} catch (error) {
logger.warn('[orchestrator-shadow] validation observation failed:', safeError(error));
throw error;
}
};
return observeWorkflowRun;
}
export const workflowShadowObserverInternals = {
recordValidationWithRetry,
safeError,
};
services/orchestrator/shadow-observer.test.mjs
+33
View File
@@ -179,6 +179,7 @@ test('shadow observer sends a control-plane-only RunSpec without user content or
userId: 'user-2',
sessionId: 'session-2',
taskType: 'code-change',
pageDataRequired: true,
userMessage: {
content: [{ type: 'text', text: 'Implement the service boundary' }],
},
@@ -192,6 +193,7 @@ test('shadow observer sends a control-plane-only RunSpec without user content or
assert.equal(body.policy.executionMode, 'observe-only');
assert.equal(body.policy.sideEffectsAllowed, false);
assert.equal(body.input.instruction, '[shadow-control-plane-only]');
assert.equal(body.input.pageDataRequired, true);
assert.equal('sessionRef' in body.input, false);
assert.deepEqual(body.subject, { tenantId: null, userId: null });
assert.equal(body.metadata.configVersion, 7);
@@ -270,6 +272,37 @@ test('isolated Portal shadow flow reaches LangGraph over HTTP and removes all te
assert.equal(checkpointJson.includes('private-user-id'), false);
assert.equal(checkpointJson.includes('private-session-id'), false);
const validation = await observer.observeValidation({
runId: 'isolated-shadow-run-1',
requestId: 'isolated-shadow-request-1',
userId: 'private-user-id',
workflowName: 'code-run-v1',
observation: {
idempotencyKey: 'isolated-shadow-run-1:page-data-delivery:v1',
taskType: 'page_data_dev_complex',
required: true,
checks: [
{ id: 'agent_run_completion', status: 'passed' },
{ id: 'page_data_binding', status: 'passed' },
{ id: 'page_data_storage_policy', status: 'passed' },
],
metrics: { pageCount: 2, publicationCount: 1 },
source: 'portal-agent-run',
observedAt: 123,
sensitive: {
workspacePath: '/private/workspace',
instruction: secretUserContent,
},
},
});
assert.equal(validation.observed, true);
assert.equal(validation.validation.verdict, 'passed');
const validatedState = await runtime.getState('isolated-shadow-run-1');
assert.equal(validatedState.validation.verdict, 'passed');
const validatedJson = JSON.stringify(validatedState.validation);
assert.equal(validatedJson.includes('/private/workspace'), false);
assert.equal(validatedJson.includes(secretUserContent), false);
const deleted = await fetch(`${server.baseUrl}/v1/runs/isolated-shadow-run-1`, {
method: 'DELETE',
headers: { authorization: 'Bearer isolated-shadow-token' },