feat: add shadow LangGraph orchestrator runtime

This commit is contained in:
john
2026-07-24 21:25:40 +08:00
parent 46ea22b342
commit 24336d0178
29 changed files with 3247 additions and 25 deletions
+16
View File
@@ -0,0 +1,16 @@
FROM node:24-bookworm-slim
ENV NODE_ENV=production
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci --omit=dev
COPY *.mjs ./
USER node
EXPOSE 8093
HEALTHCHECK --interval=15s --timeout=3s --start-period=10s --retries=3 \
CMD node -e "fetch('http://127.0.0.1:8093/ready').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))"
CMD ["node", "server.mjs"]
+70 -4
View File
@@ -29,7 +29,7 @@ Memind-facing protocol is `orchestrator-run-v1` plus `orchestrator-event-v1`.
## Current stage
Phase 1 establishes:
Phase 1 established:
- framework-neutral run and event contracts;
- a plug-in workflow engine registry;
@@ -38,9 +38,19 @@ Phase 1 establishes:
- deterministic Off / Shadow / Canary / Active routing decisions;
- an environment-level emergency kill switch.
The default mode is `off`. No production task is routed to LangGraph in this
phase. Runtime dispatch wiring and a separately deployed Orchestrator worker are
subsequent phases.
Phase 2 adds:
- a real LangGraph `StateGraph` service on the versioned HTTP boundary;
- a PostgreSQL `PostgresSaver` checkpoint owner;
- an explicitly non-durable MemorySaver option for tests and local debugging;
- a `code-run-v1` observe-only graph with no executor or tool access;
- a Portal fire-and-forget shadow projection;
- live health probing from memindadm;
- a separate Colima/Compose deployment artifact.
The default memindadm mode remains `off`. Canary and Active routing are not
wired to the LangGraph executor in Phase 2. Native Agent Run remains the sole
executor, including in Shadow mode.
## Admin configuration
@@ -55,6 +65,62 @@ Supported modes:
`MEMIND_ORCHESTRATOR_KILL_SWITCH=1` always forces Native selection.
## Local process
PostgreSQL is the default and required checkpoint backend:
```bash
MEMIND_ORCHESTRATOR_DATABASE_URL='postgresql://...' \
MEMIND_ORCHESTRATOR_SERVICE_TOKEN='...' \
pnpm dev:orchestrator
```
Memory mode is intentionally opt-in and non-durable:
```bash
MEMIND_ORCHESTRATOR_CHECKPOINT_MODE=memory pnpm dev:orchestrator
```
The service binds to `127.0.0.1:8093` by default. Configure the same URL and
service token in Portal, then enable `shadow` in `/ops/admin/orchestrator`.
## Colima deployment
Colima is the recommended first container host on macOS because this service and
its PostgreSQL checkpoint database have an isolated Compose lifecycle:
```bash
cd deploy/orchestrator
cp .env.example .env
# Replace both example secrets before starting.
docker compose up -d --build
docker compose ps
curl http://127.0.0.1:8093/ready
```
The Compose project:
- publishes only the Orchestrator port on loopback;
- does not publish PostgreSQL;
- does not mount the Docker socket or a Memind workspace;
- keeps PostgreSQL on a dedicated internal network while the service also joins
a separate edge network for its loopback-published port;
- is not part of the Goosed Compose lifecycle.
This local Compose artifact is not a production release path. Production still
requires the repository release gates and a separately approved deployment.
## Runtime environment
| Variable | Purpose | Default |
|---|---|---|
| `MEMIND_ORCHESTRATOR_HOST` | HTTP bind address | `127.0.0.1` |
| `MEMIND_ORCHESTRATOR_PORT` | HTTP port | `8093` |
| `MEMIND_ORCHESTRATOR_SERVICE_TOKEN` | Bearer token for `/v1/*` | empty |
| `MEMIND_ORCHESTRATOR_CHECKPOINT_MODE` | `postgres` or explicit `memory` | `postgres` |
| `MEMIND_ORCHESTRATOR_DATABASE_URL` | Dedicated checkpoint PostgreSQL URL | required |
| `MEMIND_ORCHESTRATOR_DATABASE_SCHEMA` | Checkpoint schema | `memind_orchestrator` |
## Extraction test
The service is ready to move into a separate repository only when:
+73 -2
View File
@@ -146,6 +146,72 @@ function engineCatalog(config) {
];
}
async function probeServiceHealth(config, {
fetchImpl = globalThis.fetch,
} = {}) {
const checkedAt = Date.now();
if (!config.serviceUrl) {
return {
checkedAt,
ok: false,
status: 'unconfigured',
latencyMs: 0,
httpStatus: null,
details: null,
};
}
if (typeof fetchImpl !== 'function') {
return {
checkedAt,
ok: false,
status: 'fetch_unavailable',
latencyMs: 0,
httpStatus: null,
details: null,
};
}
const startedAt = Date.now();
const controller = new AbortController();
const timer = setTimeout(
() => controller.abort(),
Math.max(500, Number(config.requestTimeoutMs) || 5000),
);
try {
const response = await fetchImpl(`${config.serviceUrl}/ready`, {
headers: { accept: 'application/json' },
signal: controller.signal,
});
const body = await response.json().catch(() => null);
const healthy = response.ok && body?.status === 'ok';
return {
checkedAt,
ok: healthy,
status: healthy ? 'healthy' : 'unhealthy',
latencyMs: Math.max(0, Date.now() - startedAt),
httpStatus: response.status,
details: body && typeof body === 'object' ? {
service: body.service == null ? null : String(body.service).slice(0, 128),
checkpoint: body.checkpoint && typeof body.checkpoint === 'object' ? {
kind: body.checkpoint.kind == null ? null : String(body.checkpoint.kind).slice(0, 64),
durable: Boolean(body.checkpoint.durable),
} : null,
execution: body.execution == null ? null : String(body.execution).slice(0, 64),
} : null,
};
} catch (error) {
return {
checkedAt,
ok: false,
status: error?.name === 'AbortError' ? 'timeout' : 'unreachable',
latencyMs: Math.max(0, Date.now() - startedAt),
httpStatus: null,
details: null,
};
} finally {
clearTimeout(timer);
}
}
async function ensureConfigTable(pool) {
await pool.query(`
CREATE TABLE IF NOT EXISTS ${CONFIG_TABLE} (
@@ -177,7 +243,10 @@ async function loadStoredState(pool) {
};
}
export function createOrchestratorAdminConfigService(pool, { env = process.env } = {}) {
export function createOrchestratorAdminConfigService(pool, {
env = process.env,
fetchImpl = globalThis.fetch,
} = {}) {
async function loadEffectiveState() {
const stored = await loadStoredState(pool);
if (stored) return { ...stored, source: 'admin-db' };
@@ -232,12 +301,13 @@ export function createOrchestratorAdminConfigService(pool, { env = process.env }
return this.getAdminConfig();
},
async getRuntimeState() {
async getRuntimeState({ probe = false } = {}) {
const state = await loadEffectiveState();
return {
...state,
runtime: runtimeState(state.config, env),
engines: engineCatalog(state.config),
...(probe ? { serviceHealth: await probeServiceHealth(state.config, { fetchImpl }) } : {}),
};
},
@@ -292,5 +362,6 @@ export const orchestratorAdminConfigInternals = {
CONFIG_SCOPE,
CONFIG_TABLE,
engineCatalog,
probeServiceHealth,
runtimeState,
};
@@ -97,3 +97,34 @@ test('orchestrator emergency kill switch always forces native selection', async
assert.equal(selected.engine, 'native');
assert.equal(selected.reason, 'kill_switch');
});
test('orchestrator runtime probe reports sanitized service health', async () => {
let requestedUrl = null;
const service = createOrchestratorAdminConfigService(createPool(), {
env: {},
fetchImpl: async (url) => {
requestedUrl = url;
return new Response(JSON.stringify({
status: 'ok',
service: 'memind-langgraph-orchestrator',
checkpoint: { kind: 'postgres', durable: true, password: 'must-not-pass-through' },
execution: 'observe-only',
ignoredSecret: 'must-not-pass-through',
}), {
status: 200,
headers: { 'content-type': 'application/json' },
});
},
});
await service.updateAdminConfig({
mode: 'shadow',
serviceUrl: 'http://127.0.0.1:8093',
});
const runtime = await service.getRuntimeState({ probe: true });
assert.equal(requestedUrl, 'http://127.0.0.1:8093/ready');
assert.equal(runtime.serviceHealth.ok, true);
assert.equal(runtime.serviceHealth.details.checkpoint.durable, true);
assert.equal('password' in runtime.serviceHealth.details.checkpoint, false);
assert.equal('ignoredSecret' in runtime.serviceHealth.details, false);
});
+113
View File
@@ -0,0 +1,113 @@
import express from 'express';
function errorStatus(error) {
if (Number.isInteger(error?.status)) return error.status;
if (['WORKFLOW_NOT_SUPPORTED', 'WORKFLOW_OBSERVE_ONLY_REQUIRED'].includes(error?.code)) {
return 422;
}
return 500;
}
function authorize(serviceToken) {
const expected = String(serviceToken ?? '').trim();
return (request, response, next) => {
if (!expected) return next();
if (request.get('authorization') === `Bearer ${expected}`) return next();
return response.status(401).json({
error: {
code: 'UNAUTHORIZED',
message: 'Missing or invalid service token',
},
});
};
}
export function createOrchestratorApp({
runtime,
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
} = {}) {
if (!runtime) throw new Error('Orchestrator app requires runtime');
const app = express();
app.disable('x-powered-by');
app.use(express.json({ limit: '256kb' }));
app.get('/health', (_request, response) => response.json(runtime.health()));
app.get('/ready', async (_request, response) => {
try {
return response.json(await runtime.ready());
} catch {
return response.status(503).json({
...runtime.health(),
ready: false,
});
}
});
app.use('/v1', authorize(serviceToken));
app.post('/v1/runs', async (request, response, next) => {
try {
response.status(202).json(await runtime.start(request.body));
} catch (error) {
next(error);
}
});
app.get('/v1/runs/:runId', async (request, response, next) => {
try {
const state = await runtime.getState(request.params.runId);
if (!state) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
return response.json(state);
} catch (error) {
return next(error);
}
});
app.get('/v1/runs/:runId/events', async (request, response, next) => {
try {
const result = await runtime.listEvents(request.params.runId, {
after: request.query.after,
});
if (!result) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
return response.json(result);
} catch (error) {
return next(error);
}
});
app.post('/v1/runs/:runId/resume', async (request, response, next) => {
try {
const state = await runtime.resume(request.params.runId, request.body);
if (!state) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
return response.json(state);
} catch (error) {
return next(error);
}
});
app.post('/v1/runs/:runId/cancel', async (request, response, next) => {
try {
const state = await runtime.cancel(request.params.runId, request.body);
if (!state) return response.status(404).json({ error: { code: 'RUN_NOT_FOUND' } });
return response.json(state);
} catch (error) {
return next(error);
}
});
app.use((error, _request, response, _next) => {
response.status(errorStatus(error)).json({
error: {
code: error?.code ?? 'ORCHESTRATOR_ERROR',
message: error instanceof Error ? error.message : 'Orchestrator request failed',
},
});
});
return app;
}
export const orchestratorAppInternals = {
authorize,
errorStatus,
};
+94
View File
@@ -0,0 +1,94 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { MemorySaver } from '@langchain/langgraph';
import { createOrchestratorApp } from './app.mjs';
import { createLangGraphOrchestratorRuntime } from './runtime.mjs';
async function listen(app) {
const server = await new Promise((resolve) => {
const candidate = app.listen(0, '127.0.0.1', () => resolve(candidate));
});
const address = server.address();
return {
baseUrl: `http://127.0.0.1:${address.port}`,
close: () => new Promise((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
}),
};
}
function spec() {
return {
runId: 'http-shadow-1',
requestId: 'http-request-1',
workflow: { name: 'code-run-v1', version: 1 },
input: { instruction: 'Inspect without executing' },
policy: { executionMode: 'observe-only', sideEffectsAllowed: false },
};
}
test('orchestrator HTTP API exposes health and authenticated run endpoints', async (t) => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
checkpointKind: 'memory',
});
const server = await listen(createOrchestratorApp({
runtime,
serviceToken: 'test-service-token',
}));
t.after(server.close);
const health = await fetch(`${server.baseUrl}/health`);
assert.equal(health.status, 200);
assert.equal((await health.json()).execution, 'observe-only');
const ready = await fetch(`${server.baseUrl}/ready`);
assert.equal(ready.status, 200);
assert.equal((await ready.json()).ready, true);
const unauthorized = await fetch(`${server.baseUrl}/v1/runs`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(spec()),
});
assert.equal(unauthorized.status, 401);
const created = await fetch(`${server.baseUrl}/v1/runs`, {
method: 'POST',
headers: {
authorization: 'Bearer test-service-token',
'content-type': 'application/json',
},
body: JSON.stringify(spec()),
});
assert.equal(created.status, 202);
assert.equal((await created.json()).status, 'succeeded');
const events = await fetch(`${server.baseUrl}/v1/runs/http-shadow-1/events?after=1`, {
headers: { authorization: 'Bearer test-service-token' },
});
const eventBody = await events.json();
assert.equal(events.status, 200);
assert.deepEqual(eventBody.events.map((event) => event.sequence), [2, 3]);
});
test('orchestrator HTTP API reports missing runs and observe-only violations', async (t) => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
const server = await listen(createOrchestratorApp({ runtime }));
t.after(server.close);
const missing = await fetch(`${server.baseUrl}/v1/runs/missing`);
assert.equal(missing.status, 404);
const activeSpec = spec();
activeSpec.runId = 'http-active-1';
activeSpec.policy = { executionMode: 'active', sideEffectsAllowed: true };
const rejected = await fetch(`${server.baseUrl}/v1/runs`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(activeSpec),
});
assert.equal(rejected.status, 422);
assert.equal((await rejected.json()).error.code, 'WORKFLOW_OBSERVE_ONLY_REQUIRED');
});
+74
View File
@@ -0,0 +1,74 @@
import { MemorySaver } from '@langchain/langgraph';
import { PostgresSaver } from '@langchain/langgraph-checkpoint-postgres';
const DEFAULT_POSTGRES_SCHEMA = 'memind_orchestrator';
function normalizeMode(value) {
const mode = String(value ?? 'postgres').trim().toLowerCase();
if (mode === 'memory' || mode === 'postgres') return mode;
throw new Error(`Unsupported orchestrator checkpoint mode: ${value}`);
}
function normalizeSchema(value) {
const schema = String(value ?? DEFAULT_POSTGRES_SCHEMA).trim();
if (!/^[a-z_][a-z0-9_]{0,62}$/i.test(schema)) {
throw new Error('Invalid orchestrator PostgreSQL schema');
}
return schema;
}
export async function createOrchestratorCheckpoint({
mode = process.env.MEMIND_ORCHESTRATOR_CHECKPOINT_MODE,
connectionString = process.env.MEMIND_ORCHESTRATOR_DATABASE_URL,
schema = process.env.MEMIND_ORCHESTRATOR_DATABASE_SCHEMA,
} = {}) {
const normalizedMode = normalizeMode(mode);
if (normalizedMode === 'memory') {
return {
kind: 'memory',
durable: false,
checkpointer: new MemorySaver(),
async probe() {
return true;
},
async close() {},
};
}
const normalizedConnectionString = String(connectionString ?? '').trim();
if (!normalizedConnectionString) {
const error = new Error(
'MEMIND_ORCHESTRATOR_DATABASE_URL is required when checkpoint mode is postgres',
);
error.code = 'ORCHESTRATOR_DATABASE_URL_REQUIRED';
throw error;
}
const checkpointer = PostgresSaver.fromConnString(normalizedConnectionString, {
schema: normalizeSchema(schema),
});
await checkpointer.setup();
return {
kind: 'postgres',
durable: true,
checkpointer,
async probe() {
await checkpointer.getTuple({
configurable: {
thread_id: '__orchestrator_readiness__',
checkpoint_ns: '',
},
});
return true;
},
async close() {
await checkpointer.end();
},
};
}
export const orchestratorCheckpointInternals = {
DEFAULT_POSTGRES_SCHEMA,
normalizeMode,
normalizeSchema,
};
+32
View File
@@ -0,0 +1,32 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import {
createOrchestratorCheckpoint,
orchestratorCheckpointInternals,
} from './checkpoint.mjs';
test('checkpoint defaults to durable PostgreSQL and fails closed without a URL', async () => {
await assert.rejects(
createOrchestratorCheckpoint({ mode: undefined, connectionString: '' }),
(error) => error.code === 'ORCHESTRATOR_DATABASE_URL_REQUIRED',
);
});
test('memory checkpoint is explicitly non-durable', async () => {
const checkpoint = await createOrchestratorCheckpoint({ mode: 'memory' });
assert.equal(checkpoint.kind, 'memory');
assert.equal(checkpoint.durable, false);
assert.equal(typeof checkpoint.checkpointer.getTuple, 'function');
await checkpoint.close();
});
test('checkpoint schema accepts identifiers and rejects SQL fragments', () => {
assert.equal(
orchestratorCheckpointInternals.normalizeSchema('memind_orchestrator_v2'),
'memind_orchestrator_v2',
);
assert.throws(
() => orchestratorCheckpointInternals.normalizeSchema('public; drop schema public'),
/Invalid orchestrator PostgreSQL schema/,
);
});
@@ -0,0 +1,128 @@
import {
Annotation,
END,
START,
StateGraph,
} from '@langchain/langgraph';
import {
buildRunEvent,
normalizeRunSpec,
} from './contracts.mjs';
export const CODE_RUN_SHADOW_WORKFLOW = 'code-run-v1';
const ShadowState = Annotation.Root({
spec: Annotation(),
status: Annotation({
reducer: (_current, update) => update,
default: () => 'queued',
}),
phase: Annotation({
reducer: (_current, update) => update,
default: () => 'accepted',
}),
plan: Annotation({
reducer: (_current, update) => update,
default: () => null,
}),
result: Annotation({
reducer: (_current, update) => update,
default: () => null,
}),
events: Annotation({
reducer: (current, update) => [...current, ...update],
default: () => [],
}),
});
function eventFor(state, type, data = null) {
return buildRunEvent({
runId: state.spec.runId,
sequence: state.events.length + 1,
type,
data,
});
}
function validateNode(state) {
const spec = normalizeRunSpec(state.spec);
if (spec.workflow.name !== CODE_RUN_SHADOW_WORKFLOW) {
const error = new Error(`Unsupported workflow: ${spec.workflow.name}`);
error.code = 'WORKFLOW_NOT_SUPPORTED';
throw error;
}
if (spec.policy.executionMode !== 'observe-only' || spec.policy.sideEffectsAllowed !== false) {
const error = new Error('Phase 2 LangGraph service accepts observe-only runs');
error.code = 'WORKFLOW_OBSERVE_ONLY_REQUIRED';
throw error;
}
return {
spec,
status: 'running',
phase: 'validated',
events: [eventFor(state, 'workflow_validated', {
workflow: spec.workflow,
executionMode: 'observe-only',
})],
};
}
function planNode(state) {
const taskType = String(state.spec.input?.taskType ?? '').trim() || 'code-change';
const instruction = String(state.spec.input?.instruction ?? '');
const plan = {
taskType,
executorAdapter: 'native-agent-run',
instructionCharacters: instruction.length,
steps: [
'accept_control_plane_run',
'project_executor_boundary',
'record_shadow_result',
],
};
return {
phase: 'planned',
plan,
events: [eventFor(state, 'workflow_planned', {
taskType,
executorAdapter: plan.executorAdapter,
stepCount: plan.steps.length,
})],
};
}
function finalizeNode(state) {
const result = {
observed: true,
executed: false,
executorAdapter: state.plan.executorAdapter,
taskType: state.plan.taskType,
};
return {
status: 'succeeded',
phase: 'completed',
result,
events: [eventFor(state, 'workflow_completed', result)],
};
}
export function createCodeRunShadowGraph({ checkpointer } = {}) {
if (!checkpointer) throw new Error('Code run shadow graph requires a checkpointer');
return new StateGraph(ShadowState)
.addNode('validate_run', validateNode)
.addNode('build_plan', planNode)
.addNode('finalize_run', finalizeNode)
.addEdge(START, 'validate_run')
.addEdge('validate_run', 'build_plan')
.addEdge('build_plan', 'finalize_run')
.addEdge('finalize_run', END)
.compile({ checkpointer });
}
export const codeRunShadowGraphInternals = {
ShadowState,
eventFor,
validateNode,
planNode,
finalizeNode,
};
File diff suppressed because it is too large Load Diff
+19
View File
@@ -0,0 +1,19 @@
{
"name": "@memind/workflow-orchestrator",
"private": true,
"version": "0.1.0",
"type": "module",
"engines": {
"node": ">=20"
},
"scripts": {
"start": "node server.mjs",
"test": "node --test *.test.mjs"
},
"dependencies": {
"@langchain/core": "^1.2.3",
"@langchain/langgraph": "^1.4.8",
"@langchain/langgraph-checkpoint-postgres": "^1.0.4",
"express": "^4.21.2"
}
}
+137
View File
@@ -0,0 +1,137 @@
import { normalizeRunSpec } from './contracts.mjs';
import {
CODE_RUN_SHADOW_WORKFLOW,
createCodeRunShadowGraph,
} from './code-run-shadow-graph.mjs';
function graphConfig(runId) {
return {
configurable: {
thread_id: runId,
checkpoint_ns: '',
},
};
}
function projectSnapshot(runId, snapshot) {
const values = snapshot?.values ?? {};
if (!values.spec) return null;
return {
version: 'orchestrator-state-v1',
runId,
requestId: values.spec.requestId,
workflow: values.spec.workflow,
status: values.status ?? 'unknown',
phase: values.phase ?? null,
plan: values.plan ?? null,
result: values.result ?? null,
createdAt: values.events?.[0]?.timestamp ?? null,
updatedAt: values.events?.at?.(-1)?.timestamp ?? null,
};
}
export function createLangGraphOrchestratorRuntime({
checkpointer,
checkpointKind = 'unknown',
durable = false,
checkpointProbe = null,
} = {}) {
const graph = createCodeRunShadowGraph({ checkpointer });
const probe = typeof checkpointProbe === 'function'
? checkpointProbe
: async () => {
await checkpointer.getTuple(graphConfig('__orchestrator_readiness__'));
return true;
};
async function getSnapshot(runId) {
return graph.getState(graphConfig(runId));
}
async function getState(runId) {
const normalizedRunId = String(runId ?? '').trim();
if (!normalizedRunId) return null;
return projectSnapshot(normalizedRunId, await getSnapshot(normalizedRunId));
}
function health() {
return {
status: 'ok',
service: 'memind-langgraph-orchestrator',
checkpoint: {
kind: checkpointKind,
durable: Boolean(durable),
},
execution: 'observe-only',
};
}
return {
async start(input) {
const spec = normalizeRunSpec(input);
if (spec.workflow.name !== CODE_RUN_SHADOW_WORKFLOW) {
const error = new Error(`Unsupported workflow: ${spec.workflow.name}`);
error.code = 'WORKFLOW_NOT_SUPPORTED';
error.status = 422;
throw error;
}
const existing = await getState(spec.runId);
if (existing) return existing;
await graph.invoke({
spec,
status: 'queued',
phase: 'accepted',
events: [],
}, graphConfig(spec.runId));
return getState(spec.runId);
},
getState,
async listEvents(runId, { after = 0 } = {}) {
const normalizedRunId = String(runId ?? '').trim();
const snapshot = normalizedRunId ? await getSnapshot(normalizedRunId) : null;
if (!snapshot?.values?.spec) return null;
const cursor = Math.max(0, Number(after) || 0);
const events = (snapshot.values.events ?? []).filter((event) => event.sequence > cursor);
return {
runId: normalizedRunId,
events,
nextCursor: events.at(-1)?.sequence ?? cursor,
};
},
async resume(runId) {
const state = await getState(runId);
if (!state) return null;
const error = new Error('This workflow has no interrupt point to resume');
error.code = 'WORKFLOW_NOT_INTERRUPTED';
error.status = 409;
throw error;
},
async cancel(runId) {
const state = await getState(runId);
if (!state) return null;
const error = new Error('Completed shadow observations cannot be cancelled');
error.code = 'WORKFLOW_ALREADY_TERMINAL';
error.status = 409;
throw error;
},
health,
async ready() {
await probe();
return {
...health(),
ready: true,
};
},
};
}
export const orchestratorRuntimeInternals = {
graphConfig,
projectSnapshot,
};
+88
View File
@@ -0,0 +1,88 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { MemorySaver } from '@langchain/langgraph';
import { createLangGraphOrchestratorRuntime } from './runtime.mjs';
function runSpec(overrides = {}) {
return {
runId: 'run-shadow-1',
requestId: 'request-shadow-1',
workflow: { name: 'code-run-v1', version: 1 },
subject: { userId: 'user-1' },
input: {
instruction: 'Add an idempotency test',
taskType: 'code-change',
},
policy: {
executionMode: 'observe-only',
sideEffectsAllowed: false,
},
...overrides,
};
}
test('LangGraph runtime checkpoints a deterministic observe-only code workflow', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
checkpointKind: 'memory',
durable: false,
});
const state = await runtime.start(runSpec());
assert.equal(state.runId, 'run-shadow-1');
assert.equal(state.status, 'succeeded');
assert.equal(state.phase, 'completed');
assert.equal(state.result.observed, true);
assert.equal(state.result.executed, false);
assert.equal(state.plan.executorAdapter, 'native-agent-run');
const restored = await runtime.getState('run-shadow-1');
assert.deepEqual(restored, state);
const eventPage = await runtime.listEvents('run-shadow-1');
assert.deepEqual(
eventPage.events.map((event) => event.type),
['workflow_validated', 'workflow_planned', 'workflow_completed'],
);
assert.deepEqual(
eventPage.events.map((event) => event.sequence),
[1, 2, 3],
);
assert.equal(eventPage.nextCursor, 3);
assert.equal((await runtime.listEvents('run-shadow-1', { after: 2 })).events.length, 1);
});
test('LangGraph runtime is idempotent by run id', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
const first = await runtime.start(runSpec());
const second = await runtime.start(runSpec({
input: { instruction: 'This must not replace the first checkpoint' },
}));
assert.deepEqual(second, first);
assert.equal((await runtime.listEvents('run-shadow-1')).events.length, 3);
});
test('LangGraph runtime rejects execution-enabled and unsupported workflows', async () => {
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: new MemorySaver(),
});
await assert.rejects(
runtime.start(runSpec({
runId: 'run-active',
policy: { executionMode: 'active', sideEffectsAllowed: true },
})),
(error) => error.code === 'WORKFLOW_OBSERVE_ONLY_REQUIRED',
);
await assert.rejects(
runtime.start(runSpec({
runId: 'run-unsupported',
workflow: { name: 'research-v1', version: 1 },
})),
(error) => error.code === 'WORKFLOW_NOT_SUPPORTED' && error.status === 422,
);
});
+65
View File
@@ -0,0 +1,65 @@
import { pathToFileURL } from 'node:url';
import { createOrchestratorApp } from './app.mjs';
import { createOrchestratorCheckpoint } from './checkpoint.mjs';
import { createLangGraphOrchestratorRuntime } from './runtime.mjs';
function positivePort(value, fallback = 8093) {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1 || parsed > 65535) return fallback;
return parsed;
}
export async function startOrchestratorServer({
env = process.env,
logger = console,
} = {}) {
const checkpoint = await createOrchestratorCheckpoint({
mode: env.MEMIND_ORCHESTRATOR_CHECKPOINT_MODE,
connectionString: env.MEMIND_ORCHESTRATOR_DATABASE_URL,
schema: env.MEMIND_ORCHESTRATOR_DATABASE_SCHEMA,
});
const runtime = createLangGraphOrchestratorRuntime({
checkpointer: checkpoint.checkpointer,
checkpointKind: checkpoint.kind,
durable: checkpoint.durable,
checkpointProbe: checkpoint.probe,
});
const app = createOrchestratorApp({
runtime,
serviceToken: env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
});
const host = String(env.MEMIND_ORCHESTRATOR_HOST ?? '127.0.0.1').trim() || '127.0.0.1';
const port = positivePort(env.MEMIND_ORCHESTRATOR_PORT, 8093);
const server = await new Promise((resolve, reject) => {
const listening = app.listen(port, host, () => resolve(listening));
listening.once('error', reject);
});
logger.log(`[orchestrator] listening on http://${host}:${port} (${checkpoint.kind})`);
async function close() {
await new Promise((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
await checkpoint.close();
}
return { app, server, runtime, checkpoint, close };
}
const isEntrypoint = process.argv[1]
&& import.meta.url === pathToFileURL(process.argv[1]).href;
if (isEntrypoint) {
const running = await startOrchestratorServer();
const shutdown = async (signal) => {
console.log(`[orchestrator] received ${signal}, shutting down`);
await running.close();
process.exit(0);
};
process.once('SIGINT', () => void shutdown('SIGINT'));
process.once('SIGTERM', () => void shutdown('SIGTERM'));
}
export const orchestratorServerInternals = {
positivePort,
};
+111
View File
@@ -0,0 +1,111 @@
import {
WORKFLOW_ENGINE,
normalizeRunSpec,
} from './contracts.mjs';
import { createRemoteWorkflowEngine } from './engine-registry.mjs';
const MAX_INSTRUCTION_CHARACTERS = 16_000;
function extractMessageText(message) {
if (typeof message === 'string') return message;
if (!message || typeof message !== 'object') return '';
if (typeof message.content === 'string') return message.content;
if (!Array.isArray(message.content)) return '';
return message.content
.filter((part) => part?.type === 'text')
.map((part) => String(part.text ?? ''))
.join('\n');
}
function safeError(error) {
return {
code: String(error?.code ?? 'WORKFLOW_SHADOW_FAILED').slice(0, 128),
message: String(error instanceof Error ? error.message : error).slice(0, 1000),
};
}
export function createWorkflowShadowObserver({
configService,
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
fetchImpl = globalThis.fetch,
logger = console,
} = {}) {
if (!configService?.selectEngine || !configService?.getRuntimeState) {
throw new Error('Workflow shadow observer requires orchestrator config service');
}
return async function observeWorkflowRun({
runId,
requestId,
userId,
sessionId = null,
workflowName = 'code-run-v1',
userMessage,
taskType = null,
} = {}) {
const selection = await configService.selectEngine({
runId,
requestId,
userId,
workflowName,
});
if (selection.shadowEngine !== WORKFLOW_ENGINE.LANGGRAPH) {
return {
observed: false,
reason: selection.reason,
mode: selection.mode,
};
}
const state = await configService.getRuntimeState();
const engine = createRemoteWorkflowEngine({
id: WORKFLOW_ENGINE.LANGGRAPH,
baseUrl: state.config.serviceUrl,
serviceToken,
timeoutMs: state.config.requestTimeoutMs,
fetchImpl,
});
const instruction = extractMessageText(userMessage).slice(0, MAX_INSTRUCTION_CHARACTERS);
const spec = normalizeRunSpec({
runId,
requestId,
workflow: { name: workflowName, version: 1 },
subject: { userId },
input: {
instruction,
taskType,
toolMode: 'code',
sessionRef: sessionId
? { kind: 'goose-session', id: String(sessionId) }
: null,
},
policy: {
executionMode: 'observe-only',
sideEffectsAllowed: false,
},
metadata: {
source: 'memind-agent-run',
configVersion: selection.configVersion,
},
});
try {
const result = await engine.start(spec);
return {
observed: true,
engine: WORKFLOW_ENGINE.LANGGRAPH,
mode: selection.mode,
configVersion: selection.configVersion,
shadowRun: result,
};
} catch (error) {
logger.warn('[orchestrator-shadow] observation failed:', safeError(error));
throw error;
}
};
}
export const workflowShadowObserverInternals = {
MAX_INSTRUCTION_CHARACTERS,
extractMessageText,
safeError,
};
@@ -0,0 +1,92 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { createWorkflowShadowObserver } from './shadow-observer.mjs';
function jsonResponse(body, status = 200) {
return new Response(JSON.stringify(body), {
status,
headers: { 'content-type': 'application/json' },
});
}
test('shadow observer skips without creating a remote client when mode does not select shadow', async () => {
let fetchCalls = 0;
const observer = createWorkflowShadowObserver({
configService: {
async selectEngine() {
return { shadowEngine: null, reason: 'mode_off', mode: 'off' };
},
async getRuntimeState() {
throw new Error('must not load runtime config');
},
},
fetchImpl: async () => {
fetchCalls += 1;
return jsonResponse({});
},
});
const result = await observer({
runId: 'run-1',
requestId: 'request-1',
userId: 'user-1',
});
assert.deepEqual(result, {
observed: false,
reason: 'mode_off',
mode: 'off',
});
assert.equal(fetchCalls, 0);
});
test('shadow observer sends a bounded observe-only RunSpec to LangGraph service', async () => {
let capturedUrl = null;
let capturedInit = null;
const observer = createWorkflowShadowObserver({
configService: {
async selectEngine() {
return {
shadowEngine: 'langgraph',
reason: 'shadow',
mode: 'shadow',
configVersion: 7,
};
},
async getRuntimeState() {
return {
config: {
serviceUrl: 'http://orchestrator.internal:8093',
requestTimeoutMs: 1200,
},
};
},
},
serviceToken: 'internal-token',
fetchImpl: async (url, init) => {
capturedUrl = url;
capturedInit = init;
return jsonResponse({ runId: 'run-2', status: 'succeeded' }, 202);
},
});
const result = await observer({
runId: 'run-2',
requestId: 'request-2',
userId: 'user-2',
sessionId: 'session-2',
taskType: 'code-change',
userMessage: {
content: [{ type: 'text', text: 'Implement the service boundary' }],
},
});
assert.equal(result.observed, true);
assert.equal(capturedUrl, 'http://orchestrator.internal:8093/v1/runs');
assert.equal(capturedInit.headers.authorization, 'Bearer internal-token');
const body = JSON.parse(capturedInit.body);
assert.equal(body.version, 'orchestrator-run-v1');
assert.equal(body.policy.executionMode, 'observe-only');
assert.equal(body.policy.sideEffectsAllowed, false);
assert.deepEqual(body.input.sessionRef, { kind: 'goose-session', id: 'session-2' });
assert.equal(body.metadata.configVersion, 7);
});