feat(orchestrator): harden zero-impact shadow rollout

Gate and bound Portal shadow observations while preserving Native execution. Add fail-closed service boundaries, terminal retention controls, Canary readiness telemetry, ops visibility, and isolated regression coverage.

services/orchestrator/runtime.mjs

@@ -11,6 +11,8 @@ import {
 import { createExecutorWorkerCoordinator } from './executor-worker-protocol.mjs';
 import { createExecutorAdmissionPolicy } from './executor-admission-policy.mjs';
 
+const TERMINAL_RUN_STATUSES = new Set(['succeeded', 'failed', 'cancelled']);
+
 function graphConfig(runId) {
   return {
     configurable: {
@@ -134,6 +136,35 @@ export function createLangGraphOrchestratorRuntime({
     );
   }
 
+  async function deleteRun(runId) {
+    const normalizedRunId = String(runId ?? '').trim();
+    if (!normalizedRunId) return null;
+    const state = await getState(normalizedRunId);
+    if (!state) return null;
+    if (!TERMINAL_RUN_STATUSES.has(state.status)) {
+      const error = new Error('Workflow run must be terminal before deletion');
+      error.code = 'WORKFLOW_RUN_NOT_TERMINAL';
+      error.status = 409;
+      throw error;
+    }
+    if (typeof checkpointer.deleteThread !== 'function') {
+      const error = new Error('Checkpoint backend does not support run deletion');
+      error.code = 'WORKFLOW_DELETE_UNSUPPORTED';
+      error.status = 501;
+      throw error;
+    }
+    const executorJobId = state.plan?.executorJob?.id ?? null;
+    const executorJobDeleted = executorJobId
+      ? await executorGateway.deleteJob(executorJobId)
+      : false;
+    await checkpointer.deleteThread(normalizedRunId);
+    return {
+      runId: normalizedRunId,
+      deleted: true,
+      executorJobDeleted,
+    };
+  }
+
   function health() {
     return {
       status: 'ok',
@@ -215,6 +246,77 @@ export function createLangGraphOrchestratorRuntime({
       throw error;
     },
 
+    deleteRun,
+
+    async purgeTerminalRuns({
+      before,
+      limit = 100,
+      dryRun = true,
+    } = {}) {
+      const cutoff = Number(before);
+      if (!Number.isFinite(cutoff) || cutoff <= 0) {
+        const error = new Error('Retention purge requires a positive before timestamp');
+        error.code = 'WORKFLOW_PURGE_CUTOFF_REQUIRED';
+        error.status = 422;
+        throw error;
+      }
+      const pageSize = Math.min(500, Math.max(1, Number(limit) || 100));
+      const candidates = await executorGateway.listRetentionCandidates({
+        before: cutoff,
+        limit: pageSize,
+      });
+      const uniqueCandidates = [...new Map(
+        candidates.map((candidate) => [candidate.workflowRunId, candidate]),
+      ).values()];
+      if (dryRun) {
+        return {
+          version: 'orchestrator-retention-purge-v1',
+          dryRun: true,
+          before: cutoff,
+          candidates: uniqueCandidates,
+          deleted: [],
+          failures: [],
+        };
+      }
+      const deleted = [];
+      const failures = [];
+      for (const candidate of uniqueCandidates) {
+        try {
+          const result = await deleteRun(candidate.workflowRunId);
+          if (result) {
+            deleted.push({
+              runId: candidate.workflowRunId,
+              jobId: candidate.jobId,
+            });
+            continue;
+          }
+          const executorJobDeleted = await executorGateway.deleteJob(candidate.jobId);
+          if (executorJobDeleted) {
+            deleted.push({
+              runId: candidate.workflowRunId,
+              jobId: candidate.jobId,
+              orphanedCheckpoint: true,
+            });
+          }
+        } catch (error) {
+          failures.push({
+            runId: candidate.workflowRunId,
+            jobId: candidate.jobId,
+            code: String(error?.code ?? 'WORKFLOW_PURGE_FAILED').slice(0, 128),
+            message: String(error instanceof Error ? error.message : error).slice(0, 500),
+          });
+        }
+      }
+      return {
+        version: 'orchestrator-retention-purge-v1',
+        dryRun: false,
+        before: cutoff,
+        candidates: uniqueCandidates,
+        deleted,
+        failures,
+      };
+    },
+
     getExecutorJob(jobId) {
       return executorGateway.getJob(jobId);
     },
@@ -294,4 +396,5 @@ export function createLangGraphOrchestratorRuntime({
 export const orchestratorRuntimeInternals = {
   graphConfig,
   projectSnapshot,
+  TERMINAL_RUN_STATUSES,
 };