tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: add orchestrator canary readiness gate

Browse Source
This commit is contained in:
john
2026-07-24 22:09:19 +08:00
parent ff9c68e57f
commit 85e888b340
8 changed files with 534 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/orchestrator/observability.mjs
+194 -4
View File
@@ -5,6 +5,18 @@ const SHADOW_EVENT_TYPES = Object.freeze([
'workflow_shadow_failed',
]);
const MAX_METRIC_EVENTS = 5000;
const CANARY_READINESS_THRESHOLDS = Object.freeze({
hours: 24 * 7,
minObservations: 20,
minSuccessRate: 0.95,
maxP95LatencyMs: 2000,
minLatencyCoverageRate: 0.95,
minNativeSettledRate: 0.95,
minDistinctSessions: 5,
maxHoursSinceLastObservation: 24,
});
const NATIVE_TERMINAL_STATUSES = new Set(['succeeded', 'failed']);
const SYNTHETIC_TASK_TYPES = new Set(['orchestrator_shadow_smoke']);
function clampInteger(value, fallback, min, max) {
const parsed = Number(value);
@@ -38,6 +50,7 @@ function projectShadowEventRow(row) {
const data = parseJsonColumn(row.data_json, {}) ?? {};
const succeeded = row.event_type === 'workflow_shadow_completed';
const latency = Number(data.latencyMs);
const taskType = data.taskType ?? null;
return {
eventId: row.event_id,
runId: row.run_id,
@@ -50,7 +63,8 @@ function projectShadowEventRow(row) {
engine: data.engine ?? 'langgraph',
configVersion: data.configVersion == null ? null : Number(data.configVersion),
phase: data.phase ?? null,
taskType: data.taskType ?? null,
taskType,
synthetic: data.synthetic === true || SYNTHETIC_TASK_TYPES.has(String(taskType ?? '')),
executorAdapter: data.executorAdapter ?? null,
latencyMs: Number.isFinite(latency) && latency >= 0 ? latency : null,
error: succeeded ? null : {
@@ -64,6 +78,170 @@ function projectShadowEventRow(row) {
};
}
function projectUniqueShadowRuns(rows) {
const seen = new Set();
const runs = [];
for (const row of rows) {
const run = projectShadowEventRow(row);
if (seen.has(run.runId)) continue;
seen.add(run.runId);
runs.push(run);
}
return runs;
}
function ratio(numerator, denominator) {
return denominator > 0 ? numerator / denominator : null;
}
function buildCanaryReadiness(loaded, runtimeState, now) {
const thresholds = CANARY_READINESS_THRESHOLDS;
const eligibleRuns = loaded.runs.filter((run) => !run.synthetic);
const successes = eligibleRuns.filter((run) => run.shadowStatus === 'succeeded').length;
const latencies = eligibleRuns
.map((run) => run.latencyMs)
.filter((value) => Number.isFinite(value));
const nativeSettled = eligibleRuns.filter((run) =>
NATIVE_TERMINAL_STATUSES.has(run.nativeStatus)).length;
const distinctSessions = new Set(
eligibleRuns.map((run) => run.sessionId).filter(Boolean),
).size;
const lastObservedAt = eligibleRuns[0]?.observedAt ?? null;
const hoursSinceLastObservation = lastObservedAt == null
? null
: Math.max(0, (now - lastObservedAt) / (60 * 60 * 1000));
const successRate = ratio(successes, eligibleRuns.length);
const latencyCoverageRate = ratio(latencies.length, eligibleRuns.length);
const nativeSettledRate = ratio(nativeSettled, eligibleRuns.length);
const latencyP95Ms = percentile(latencies, 0.95);
const serviceHealth = runtimeState?.serviceHealth ?? null;
const checkpoint = serviceHealth?.details?.checkpoint ?? null;
const checks = [
{
id: 'shadow_mode',
passed: runtimeState?.config?.mode === 'shadow',
actual: runtimeState?.config?.mode ?? null,
target: 'shadow',
},
{
id: 'service_healthy',
passed: serviceHealth?.ok === true,
actual: serviceHealth?.status ?? null,
target: 'healthy',
},
{
id: 'durable_checkpoint',
passed: checkpoint?.durable === true,
actual: checkpoint?.durable ?? null,
target: true,
},
{
id: 'observe_only',
passed: serviceHealth?.details?.execution === 'observe-only',
actual: serviceHealth?.details?.execution ?? null,
target: 'observe-only',
},
{
id: 'sample_volume',
passed: eligibleRuns.length >= thresholds.minObservations,
actual: eligibleRuns.length,
target: thresholds.minObservations,
},
{
id: 'shadow_success_rate',
passed: successRate != null && successRate >= thresholds.minSuccessRate,
actual: successRate,
target: thresholds.minSuccessRate,
},
{
id: 'latency_coverage',
passed: latencyCoverageRate != null
&& latencyCoverageRate >= thresholds.minLatencyCoverageRate,
actual: latencyCoverageRate,
target: thresholds.minLatencyCoverageRate,
},
{
id: 'latency_p95',
passed: latencyP95Ms != null && latencyP95Ms <= thresholds.maxP95LatencyMs,
actual: latencyP95Ms,
target: thresholds.maxP95LatencyMs,
},
{
id: 'native_settled_rate',
passed: nativeSettledRate != null
&& nativeSettledRate >= thresholds.minNativeSettledRate,
actual: nativeSettledRate,
target: thresholds.minNativeSettledRate,
},
{
id: 'session_coverage',
passed: distinctSessions >= thresholds.minDistinctSessions,
actual: distinctSessions,
target: thresholds.minDistinctSessions,
},
{
id: 'sample_freshness',
passed: hoursSinceLastObservation != null
&& hoursSinceLastObservation <= thresholds.maxHoursSinceLastObservation,
actual: hoursSinceLastObservation,
target: thresholds.maxHoursSinceLastObservation,
},
{
id: 'complete_window',
passed: !loaded.capped,
actual: loaded.capped ? 'sampled' : 'complete',
target: 'complete',
},
];
const failureCounts = new Map();
for (const run of eligibleRuns) {
if (!run.error?.code) continue;
failureCounts.set(run.error.code, (failureCounts.get(run.error.code) ?? 0) + 1);
}
return {
generatedAt: now,
ready: checks.every((check) => check.passed),
recommendation: checks.every((check) => check.passed)
? 'manual_canary_review'
: 'keep_shadow',
window: {
hours: loaded.hours,
from: loaded.from,
},
thresholds,
samples: {
totalObservations: loaded.runs.length,
eligibleObservations: eligibleRuns.length,
excludedSynthetic: loaded.runs.length - eligibleRuns.length,
successes,
failures: eligibleRuns.length - successes,
successRate,
latencyCoverageRate,
latencyP95Ms,
nativeSettledRate,
distinctSessions,
lastObservedAt,
hoursSinceLastObservation,
sampled: loaded.capped,
},
service: {
status: serviceHealth?.status ?? null,
latencyMs: serviceHealth?.latencyMs ?? null,
checkpointKind: checkpoint?.kind ?? null,
checkpointDurable: checkpoint?.durable ?? null,
execution: serviceHealth?.details?.execution ?? null,
},
checks,
blockers: checks.filter((check) => !check.passed).map((check) => check.id),
failureCodes: [...failureCounts.entries()]
.map(([code, count]) => ({ code, count }))
.sort((left, right) => right.count - left.count || left.code.localeCompare(right.code))
.slice(0, 10),
};
}
function summarizeRuns(runs, { capped = false } = {}) {
const successes = runs.filter((run) => run.shadowStatus === 'succeeded').length;
const failures = runs.length - successes;
@@ -97,6 +275,7 @@ export function createOrchestratorObservabilityService({
configService,
serviceToken = process.env.MEMIND_ORCHESTRATOR_SERVICE_TOKEN,
fetchImpl = globalThis.fetch,
nowMs = Date.now,
} = {}) {
if (!pool?.query) throw new Error('Orchestrator observability requires a database pool');
if (!configService?.getRuntimeState) {
@@ -105,7 +284,7 @@ export function createOrchestratorObservabilityService({
async function loadShadowEvents({ hours = 24 } = {}) {
const normalizedHours = clampInteger(hours, 24, 1, 24 * 30);
const from = Date.now() - normalizedHours * 60 * 60 * 1000;
const from = nowMs() - normalizedHours * 60 * 60 * 1000;
const [rows] = await pool.query(
`SELECT
e.id AS event_id,
@@ -131,11 +310,19 @@ export function createOrchestratorObservabilityService({
from,
hours: normalizedHours,
capped: rows.length >= MAX_METRIC_EVENTS,
runs: rows.map(projectShadowEventRow),
runs: projectUniqueShadowRuns(rows),
};
}
return {
async getCanaryReadiness() {
const [loaded, runtimeState] = await Promise.all([
loadShadowEvents({ hours: CANARY_READINESS_THRESHOLDS.hours }),
configService.getRuntimeState({ probe: true }),
]);
return buildCanaryReadiness(loaded, runtimeState, nowMs());
},
async listShadowRuns({
hours = 24,
limit = 50,
@@ -148,7 +335,7 @@ export function createOrchestratorObservabilityService({
? loaded.runs
: loaded.runs.filter((run) => run.shadowStatus === normalizedStatus);
return {
generatedAt: Date.now(),
generatedAt: nowMs(),
window: {
hours: loaded.hours,
from: loaded.from,
@@ -244,11 +431,14 @@ export function createOrchestratorObservabilityService({
}
export const orchestratorObservabilityInternals = {
CANARY_READINESS_THRESHOLDS,
MAX_METRIC_EVENTS,
SHADOW_EVENT_TYPES,
buildCanaryReadiness,
normalizeRunId,
parseJsonColumn,
percentile,
projectShadowEventRow,
projectUniqueShadowRuns,
summarizeRuns,
};