feat: expose executor job observability

2026-07-24 23:31:29 +08:00
parent bbb43f97a3
commit 396bb78200
18 changed files with 733 additions and 68 deletions
@@ -48,6 +48,11 @@ function normalizeRunId(value) {
  return /^[a-zA-Z0-9_-]{1,128}$/.test(runId) ? runId : '';
 }

+function normalizeExecutorJobId(value) {
+  const jobId = String(value ?? '').trim();
+  return /^[a-zA-Z0-9:_-]{1,128}$/.test(jobId) ? jobId : '';
+}
+
 function projectShadowEventRow(row) {
  const data = parseJsonColumn(row.data_json, {}) ?? {};
  const succeeded = row.event_type === 'workflow_shadow_completed';
@@ -159,6 +164,7 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
  const latencyP95Ms = percentile(latencies, 0.95);
  const serviceHealth = runtimeState?.serviceHealth ?? null;
  const checkpoint = serviceHealth?.details?.checkpoint ?? null;
+  const executorStore = serviceHealth?.details?.executorGateway?.store ?? null;

  const checks = [
    {
@@ -179,6 +185,12 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
      actual: checkpoint?.durable ?? null,
      target: true,
    },
+    {
+      id: 'durable_executor_job_store',
+      passed: executorStore?.durable === true,
+      actual: executorStore?.durable ?? null,
+      target: true,
+    },
    {
      id: 'observe_only',
      passed: serviceHealth?.details?.execution === 'observe-only',
@@ -274,6 +286,8 @@ function buildCanaryReadiness(loaded, runtimeState, now) {
      latencyMs: serviceHealth?.latencyMs ?? null,
      checkpointKind: checkpoint?.kind ?? null,
      checkpointDurable: checkpoint?.durable ?? null,
+      executorJobStoreKind: executorStore?.kind ?? null,
+      executorJobStoreDurable: executorStore?.durable ?? null,
      execution: serviceHealth?.details?.execution ?? null,
    },
    checks,
@@ -513,6 +527,12 @@ export function createOrchestratorObservabilityService({
        state: null,
        events: [],
        error: null,
+        executorJob: {
+          available: false,
+          state: null,
+          events: [],
+          error: null,
+        },
      };
      try {
        const runtimeState = await configService.getRuntimeState();
@@ -534,11 +554,42 @@ export function createOrchestratorObservabilityService({
              return collected;
            })(),
          ]);
+          const executorJobId = normalizeExecutorJobId(state?.plan?.executorJob?.id);
+          let executorJob = {
+            available: false,
+            state: null,
+            events: [],
+            error: null,
+          };
+          if (executorJobId) {
+            try {
+              const [jobState, jobEvents] = await Promise.all([
+                engine.getExecutorJob(executorJobId),
+                (async () => {
+                  const collected = [];
+                  for await (const event of engine.streamExecutorJobEvents(executorJobId)) {
+                    collected.push(event);
+                    if (collected.length >= 500) break;
+                  }
+                  return collected;
+                })(),
+              ]);
+              executorJob = {
+                available: true,
+                state: jobState,
+                events: jobEvents,
+                error: null,
+              };
+            } catch (error) {
+              executorJob.error = safeRemoteError(error);
+            }
+          }
          remote = {
            available: true,
            state,
            events,
            error: null,
+            executorJob,
          };
        }
      } catch (error) {
@@ -572,6 +623,7 @@ export const orchestratorObservabilityInternals = {
  MAX_METRIC_EVENTS,
  SHADOW_EVENT_TYPES,
  buildCanaryReadiness,
+  normalizeExecutorJobId,
  normalizeRunId,
  parseJsonColumn,
  percentile,