tkmind/memind
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

feat: harden orchestrator execution runtime

Browse Source
This commit is contained in:
john
2026-07-24 23:53:32 +08:00
parent 396bb78200
commit f6f2cd0933
55 changed files with 5122 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/orchestrator/executor-job-store.mjs
+272 -22
View File
@@ -5,6 +5,7 @@ const { Pool } = pg;
const DEFAULT_EXECUTOR_SCHEMA = 'memind_orchestrator';
const EXECUTOR_JOB_TABLE = 'executor_jobs';
const EXECUTOR_JOB_EVENT_TABLE = 'executor_job_events';
const EXECUTOR_WORKER_TABLE = 'executor_workers';
function normalizeMode(value) {
const mode = String(value ?? 'postgres').trim().toLowerCase();
@@ -46,6 +47,31 @@ function eventWithSequence(event, jobId, sequence) {
};
}
async function appendEvent(client, eventTable, jobId, event) {
if (!event) return null;
const sequenceResult = await client.query(
`SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence
FROM ${eventTable}
WHERE job_id = $1`,
[jobId],
);
const sequence = Number(sequenceResult.rows?.[0]?.next_sequence ?? 1);
const persistedEvent = eventWithSequence(event, jobId, sequence);
await client.query(
`INSERT INTO ${eventTable}
(event_id, job_id, sequence, event_json, created_at)
VALUES ($1, $2, $3, $4::jsonb, $5)`,
[
persistedEvent.eventId,
jobId,
sequence,
JSON.stringify(persistedEvent),
persistedEvent.timestamp,
],
);
return persistedEvent;
}
async function withTransaction(pool, operation) {
if (typeof pool?.connect !== 'function') {
throw new Error('PostgreSQL Executor Job Store requires transactional pool.connect');
@@ -76,6 +102,7 @@ export function createPostgresExecutorJobStore({
const normalizedSchema = normalizeSchema(schema);
const table = `"${normalizedSchema}"."${EXECUTOR_JOB_TABLE}"`;
const eventTable = `"${normalizedSchema}"."${EXECUTOR_JOB_EVENT_TABLE}"`;
const workerTable = `"${normalizedSchema}"."${EXECUTOR_WORKER_TABLE}"`;
return {
kind: 'postgres',
@@ -136,6 +163,17 @@ export function createPostgresExecutorJobStore({
)
ON CONFLICT DO NOTHING
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS ${workerTable} (
worker_id VARCHAR(128) PRIMARY KEY,
state_json JSONB NOT NULL,
last_seen_at BIGINT NOT NULL
)
`);
await pool.query(`
CREATE INDEX IF NOT EXISTS executor_workers_last_seen_idx
ON ${workerTable} (last_seen_at DESC)
`);
},
async getById(jobId) {
@@ -218,32 +256,242 @@ export function createPostgresExecutorJobStore({
RETURNING state_json`,
[record.jobId, JSON.stringify(record), record.updatedAt],
);
if (event) {
const sequenceResult = await client.query(
`SELECT COALESCE(MAX(sequence), 0) + 1 AS next_sequence
FROM ${eventTable}
WHERE job_id = $1`,
[record.jobId],
);
const sequence = Number(sequenceResult.rows?.[0]?.next_sequence ?? 1);
const persistedEvent = eventWithSequence(event, record.jobId, sequence);
await client.query(
`INSERT INTO ${eventTable}
(event_id, job_id, sequence, event_json, created_at)
VALUES ($1, $2, $3, $4::jsonb, $5)`,
[
persistedEvent.eventId,
record.jobId,
sequence,
JSON.stringify(persistedEvent),
persistedEvent.timestamp,
],
);
}
await appendEvent(client, eventTable, record.jobId, event);
return projectRow(result.rows?.[0]);
});
},
async claimNext({
workerId,
executors,
now,
leaseToken,
leaseDurationMs,
eventFactory,
}) {
return withTransaction(pool, async (client) => {
const selected = await client.query(
`SELECT state_json
FROM ${table}
WHERE (
state_json->>'status' = 'queued'
OR (
state_json->>'status' = 'retryable'
AND COALESCE((state_json->>'nextAttemptAt')::BIGINT, 0) <= $1
)
)
AND state_json->>'executor' = ANY($2::text[])
ORDER BY created_at ASC, job_id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1`,
[now, executors],
);
const current = projectRow(selected.rows?.[0]);
if (!current) return null;
const updated = {
...current,
status: 'leased',
reason: null,
attempts: Number(current.attempts ?? 0) + 1,
nextAttemptAt: null,
lease: {
token: leaseToken,
workerId,
acquiredAt: now,
heartbeatAt: now,
expiresAt: now + leaseDurationMs,
},
updatedAt: now,
completedAt: null,
};
const result = await client.query(
`UPDATE ${table}
SET state_json = $2::jsonb,
updated_at = $3
WHERE job_id = $1
RETURNING state_json`,
[updated.jobId, JSON.stringify(updated), updated.updatedAt],
);
await appendEvent(
client,
eventTable,
updated.jobId,
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
);
return projectRow(result.rows?.[0]);
});
},
async transition(jobId, {
expectedStatuses = [],
leaseToken = null,
updater,
eventFactory = null,
} = {}) {
return withTransaction(pool, async (client) => {
const locked = await client.query(
`SELECT state_json FROM ${table} WHERE job_id = $1 FOR UPDATE`,
[String(jobId ?? '').trim()],
);
const current = projectRow(locked.rows?.[0]);
if (!current) return { applied: false, record: null, reason: 'not_found' };
if (expectedStatuses.length && !expectedStatuses.includes(current.status)) {
return { applied: false, record: current, reason: 'status_mismatch' };
}
if (leaseToken != null && current.lease?.token !== leaseToken) {
return { applied: false, record: current, reason: 'lease_mismatch' };
}
const updated = updater(structuredClone(current));
if (!updated || updated.jobId !== current.jobId) {
throw new Error('Executor job transition must preserve job identity');
}
const result = await client.query(
`UPDATE ${table}
SET state_json = $2::jsonb,
updated_at = $3
WHERE job_id = $1
RETURNING state_json`,
[current.jobId, JSON.stringify(updated), updated.updatedAt],
);
await appendEvent(
client,
eventTable,
current.jobId,
typeof eventFactory === 'function' ? eventFactory(updated, current) : null,
);
return {
applied: true,
record: projectRow(result.rows?.[0]),
reason: null,
};
});
},
async listExpiredLeases({ now, limit = 100 } = {}) {
const pageSize = Math.min(500, Math.max(1, Number(limit) || 100));
const result = await pool.query(
`SELECT state_json
FROM ${table}
WHERE state_json->>'status' = ANY($1::text[])
AND COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) <= $2
ORDER BY COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) ASC
LIMIT $3`,
[['leased', 'running'], now, pageSize],
);
return (result.rows ?? []).map(projectRow).filter(Boolean);
},
async getQueueStats({ now = Date.now() } = {}) {
const result = await pool.query(
`SELECT
COUNT(*)::BIGINT AS total,
COUNT(*) FILTER (
WHERE state_json->>'status' = 'queued'
OR (
state_json->>'status' = 'retryable'
AND COALESCE((state_json->>'nextAttemptAt')::BIGINT, 0) <= $1
)
)::BIGINT AS claimable,
COUNT(*) FILTER (
WHERE state_json->>'status' = ANY($2::text[])
AND COALESCE((state_json->'lease'->>'expiresAt')::BIGINT, 0) <= $1
)::BIGINT AS expired_leases,
state_json->>'status' AS status,
COUNT(*)::BIGINT AS status_count
FROM ${table}
GROUP BY state_json->>'status'`,
[now, ['leased', 'running']],
);
const counts = {};
let total = 0;
let claimable = 0;
let expiredLeases = 0;
for (const row of result.rows ?? []) {
if (row.status) counts[row.status] = Number(row.status_count ?? 0);
total += Number(row.status_count ?? 0);
claimable += Number(row.claimable ?? 0);
expiredLeases += Number(row.expired_leases ?? 0);
}
return { total, counts, claimable, expiredLeases };
},
async getAdmissionStats({
tenantId = null,
userId = null,
now = Date.now(),
windowMs = 60_000,
} = {}) {
const result = await pool.query(
`SELECT
COUNT(*) FILTER (
WHERE state_json->>'status' = ANY($1::text[])
)::BIGINT AS global_active,
COUNT(*) FILTER (
WHERE state_json->>'status' = ANY($1::text[])
AND $2::text IS NOT NULL
AND state_json->'request'->'subject'->>'tenantId' = $2
)::BIGINT AS tenant_active,
COUNT(*) FILTER (
WHERE state_json->>'status' = ANY($1::text[])
AND $3::text IS NOT NULL
AND state_json->'request'->'subject'->>'userId' = $3
)::BIGINT AS user_active,
COUNT(*) FILTER (
WHERE $2::text IS NOT NULL
AND state_json->'request'->'subject'->>'tenantId' = $2
AND created_at >= $4
)::BIGINT AS tenant_recent,
COUNT(*) FILTER (
WHERE $3::text IS NOT NULL
AND state_json->'request'->'subject'->>'userId' = $3
AND created_at >= $4
)::BIGINT AS user_recent
FROM ${table}`,
[
['queued', 'retryable', 'leased', 'running'],
tenantId,
userId,
now - windowMs,
],
);
const row = result.rows?.[0] ?? {};
return {
globalActive: Number(row.global_active ?? 0),
tenantActive: Number(row.tenant_active ?? 0),
userActive: Number(row.user_active ?? 0),
tenantRecent: Number(row.tenant_recent ?? 0),
userRecent: Number(row.user_recent ?? 0),
};
},
async recordWorkerHeartbeat(record) {
await pool.query(
`INSERT INTO ${workerTable} (worker_id, state_json, last_seen_at)
VALUES ($1, $2::jsonb, $3)
ON CONFLICT (worker_id) DO UPDATE
SET state_json = EXCLUDED.state_json,
last_seen_at = EXCLUDED.last_seen_at`,
[record.workerId, JSON.stringify(record), record.lastSeenAt],
);
return structuredClone(record);
},
async listWorkers({ now = Date.now(), staleAfterMs = 60_000 } = {}) {
const result = await pool.query(
`SELECT state_json
FROM ${workerTable}
ORDER BY last_seen_at DESC
LIMIT 500`,
);
return (result.rows ?? [])
.map(projectRow)
.filter(Boolean)
.map((worker) => ({
...worker,
stale: Number(worker.lastSeenAt ?? 0) < now - staleAfterMs,
}));
},
async listEvents(jobId, { after = 0, limit = 500 } = {}) {
const cursor = Math.max(0, Number(after) || 0);
const pageSize = Math.min(500, Math.max(1, Number(limit) || 500));
@@ -320,6 +568,8 @@ export const executorJobStoreInternals = {
DEFAULT_EXECUTOR_SCHEMA,
EXECUTOR_JOB_TABLE,
EXECUTOR_JOB_EVENT_TABLE,
EXECUTOR_WORKER_TABLE,
appendEvent,
eventWithSequence,
normalizeMode,
normalizeSchema,