import { spawn } from 'node:child_process'; import fs from 'node:fs/promises'; import path from 'node:path'; import { assertPortalRuntimePath, hashArtifact, inspectPortalRuntime } from './artifact.mjs'; import { loadScenarioCatalog } from './catalog.mjs'; import { AUTOMATION_SUITES, validateAutomationSuites } from './coverage.mjs'; import { selectImpactScenarios } from './impact.mjs'; import { loadActiveRegressionCorpus } from './regression-corpus.mjs'; import { buildIncrementalReport, findCarryForwardBaseline, isReleaseScenario, mergeIncrementalScenarios, } from './incremental.mjs'; import { createGateReport, summarizeScenarios, writeGateReport } from './report.mjs'; import { assertSafeGateEnvironment, assertSafePortalBase } from './safety.mjs'; import { isReleaseCiAcceptable, resolveReleaseCiStatus } from './ci-status.mjs'; const ROOT = path.resolve(new URL('..', import.meta.url).pathname); const MODES = new Set([ 'deterministic', 'scenarios', 'browser', 'providers', 'upgrade', 'all', 'impact', 'incremental', ]); async function git(...args) { const result = await runCommand('git', args, { cwd: ROOT, timeoutMs: 30_000 }); if (result.code !== 0) throw new Error(`git ${args.join(' ')} failed`); return result.stdout.trim(); } export function parseRunnerArgs(argv) { const options = { mode: 'all', artifact: null, portalBase: 'http://127.0.0.1:8081', reportRoot: path.join(ROOT, '.release-gate'), timeoutMs: 15 * 60 * 1000, suiteConcurrency: Number(process.env.RELEASE_GATE_SUITE_CONCURRENCY ?? 4), baselineCommit: null, deployedCommit: null, }; for (let index = 2; index < argv.length; index += 1) { const arg = argv[index]; if (arg === '--mode') options.mode = argv[++index] ?? ''; else if (arg === '--artifact') options.artifact = path.resolve(argv[++index] ?? ''); else if (arg === '--portal-base') options.portalBase = argv[++index] ?? ''; else if (arg === '--report-root') options.reportRoot = path.resolve(argv[++index] ?? ''); else if (arg === '--timeout-ms') options.timeoutMs = Number(argv[++index]); else if (arg === '--suite-concurrency') options.suiteConcurrency = Number(argv[++index]); else if (arg === '--baseline-commit') options.baselineCommit = argv[++index] ?? ''; else if (arg === '--deployed-commit') options.deployedCommit = argv[++index] ?? ''; else if (arg === '-h' || arg === '--help') options.help = true; else throw new Error(`Unknown argument: ${arg}`); } if (!MODES.has(options.mode)) throw new Error(`Invalid release gate mode: ${options.mode}`); if (!Number.isFinite(options.timeoutMs) || options.timeoutMs <= 0) { throw new Error('timeout-ms must be a positive number'); } if ( !Number.isFinite(options.suiteConcurrency) || options.suiteConcurrency <= 0 || options.suiteConcurrency > 16 ) { throw new Error('suite-concurrency must be between 1 and 16'); } options.suiteConcurrency = Math.floor(options.suiteConcurrency); return options; } export async function runWithConcurrency(items, concurrency, worker) { const results = new Array(items.length); let nextIndex = 0; async function runWorker() { while (nextIndex < items.length) { const index = nextIndex; nextIndex += 1; results[index] = await worker(items[index], index); } } await Promise.all( Array.from( { length: Math.min(items.length, Math.max(1, concurrency)) }, () => runWorker(), ), ); return results; } // Some suites intentionally mutate the shared candidate artifact. Run those // suites before the parallel batch so they cannot race suites that boot the // same runtime while it is being rebuilt. export async function runSuitesWithConcurrency(items, concurrency, worker) { const results = new Array(items.length); const parallel = []; for (let index = 0; index < items.length; index += 1) { if (items[index].exclusive) { results[index] = await worker(items[index], index); } else { parallel.push({ item: items[index], index }); } } const parallelResults = await runWithConcurrency( parallel, concurrency, ({ item, index }) => worker(item, index), ); for (let index = 0; index < parallel.length; index += 1) { results[parallel[index].index] = parallelResults[index]; } return results; } export function filterGeneratedWorktreeStatus(status) { return status .split('\n') .filter((line) => { if (!line) return false; const filePath = line.slice(3).replace(/^"|"$/g, ''); return !filePath.startsWith('.release-gate/') && !filePath.startsWith('.runtime/'); }) .join('\n'); } export async function runCommand(command, args, { cwd = ROOT, timeoutMs = 15 * 60 * 1000, env = process.env, } = {}) { return new Promise((resolve) => { const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], }); let stdout = ''; let stderr = ''; let timedOut = false; const timer = setTimeout(() => { timedOut = true; child.kill('SIGTERM'); }, timeoutMs); child.stdout.on('data', (chunk) => { stdout += chunk; }); child.stderr.on('data', (chunk) => { stderr += chunk; }); child.on('error', (error) => { clearTimeout(timer); resolve({ code: 1, stdout, stderr: `${stderr}${error.message}`, timedOut }); }); child.on('close', (code) => { clearTimeout(timer); resolve({ code: code ?? 1, stdout, stderr, timedOut }); }); }); } function scenarioResult(scenario) { return { id: scenario.id, name: scenario.name, status: 'unknown', cleanup_status: 'not_required', evidence: [], reason: 'automation_not_implemented', }; } async function runSuite(suite, outputDir, timeoutMs) { const [command, ...args] = suite.command; const result = await runCommand(command, args, { cwd: ROOT, timeoutMs }); const logDir = path.join(outputDir, 'logs'); await fs.mkdir(logDir, { recursive: true }); const logPath = path.join(logDir, `${suite.id}.log`); await fs.writeFile( logPath, [ `$ ${[command, ...args].join(' ')}`, `exit_code=${result.code}`, `timed_out=${result.timedOut}`, '', result.stdout, result.stderr, ].join('\n'), ); return { ...result, logPath: path.relative(outputDir, logPath) }; } async function applyRepositoryChecks(results, artifactPath) { const byId = new Map(results.map((result) => [result.id, result])); const branch = await git('branch', '--show-current'); const status = await git('status', '--porcelain=v1', '--untracked-files=all'); const relevantStatus = filterGeneratedWorktreeStatus(status); const originMain = await runCommand( 'git', ['rev-parse', '--verify', 'origin/main'], { cwd: ROOT, timeoutMs: 30_000 }, ); const remoteSha = originMain.code === 0 ? originMain.stdout.trim() : null; const headSha = await git('rev-parse', 'HEAD'); const rel01 = byId.get('REL-01'); rel01.status = branch === 'main' && relevantStatus === '' ? 'passed' : 'failed'; rel01.reason = rel01.status === 'passed' ? null : 'candidate_must_be_clean_main'; rel01.evidence.push( `branch=${branch || 'detached'}`, `worktree_clean=${relevantStatus === ''}`, `ignored_generated_paths=${status !== relevantStatus}`, ); let ahead = '0'; let behind = '0'; if (remoteSha) { const aheadBehind = await runCommand( 'git', ['rev-list', '--left-right', '--count', `HEAD...${remoteSha}`], { cwd: ROOT, timeoutMs: 30_000 }, ); if (aheadBehind.code === 0) { [ahead, behind] = aheadBehind.stdout.trim().split(/\s+/); } } const ciStatus = remoteSha && behind === '0' ? await resolveReleaseCiStatus(headSha) : 'missing'; const rel02 = byId.get('REL-02'); const mainlineReady = Boolean(remoteSha) && behind === '0' && (remoteSha === headSha || Number(ahead) > 0); const ciAcceptable = isReleaseCiAcceptable(ciStatus, { headSha, remoteSha }); rel02.status = mainlineReady && ciAcceptable ? 'passed' : 'failed'; rel02.reason = rel02.status === 'passed' ? null : 'candidate_must_equal_origin_main_with_acceptable_ci'; rel02.evidence.push( `head=${headSha}`, `origin_main=${remoteSha ?? 'missing'}`, `ahead=${ahead}`, `behind=${behind}`, `ci_status=${ciStatus || 'missing'}`, ); if (!artifactPath) return null; const artifact = await hashArtifact(artifactPath); const rel03 = byId.get('REL-03'); if (rel03) { rel03.reason = 'repeat_build_comparison_not_implemented'; rel03.evidence.push(`artifact_sha256=${artifact.sha256}`, `artifact_kind=${artifact.kind}`); } const inspection = await inspectPortalRuntime(artifactPath); const rel04 = byId.get('REL-04'); rel04.status = inspection.passed ? 'passed' : 'failed'; rel04.reason = rel04.status === 'passed' ? null : 'artifact_incomplete_or_contains_forbidden_paths'; rel04.evidence.push( `missing=${inspection.missing.join(',') || 'none'}`, `forbidden=${inspection.forbidden.join(',') || 'none'}`, ); const rel11 = byId.get('REL-11'); rel11.status = inspection.missing.length === 0 ? 'unknown' : 'failed'; rel11.reason = rel11.status === 'failed' ? 'runtime_dependency_closure_incomplete' : 'runtime_container_start_smoke_not_implemented'; rel11.evidence.push(`missing=${inspection.missing.join(',') || 'none'}`); return artifact; } function suiteIsInMode(suite, mode) { return mode === 'all' || suite.mode === mode; } export async function executeReleaseGate(options) { const startedAt = new Date(); assertSafeGateEnvironment({ targets: [options.portalBase] }); assertSafePortalBase(options.portalBase); if (options.artifact) { options.artifact = assertPortalRuntimePath(options.artifact, { repoRoot: ROOT }); } const catalog = await loadScenarioCatalog({ root: ROOT }); validateAutomationSuites(catalog); const commitSha = await git('rev-parse', 'HEAD'); const branch = await git('branch', '--show-current'); const outputDir = options.mode === 'all' ? path.join(options.reportRoot, commitSha) : path.join(options.reportRoot, commitSha, 'partials', options.mode); const results = catalog.map(scenarioResult); const byId = new Map(results.map((result) => [result.id, result])); const artifact = await applyRepositoryChecks(results, options.artifact); const regressionCorpus = await loadActiveRegressionCorpus({ root: ROOT, catalog }); const comp09 = byId.get('COMP-09'); comp09.reason = regressionCorpus.status === 'ready' ? 'production_regression_replay_not_implemented' : `production_regression_corpus_${regressionCorpus.status}`; comp09.evidence.push( `manifest=${path.relative(ROOT, regressionCorpus.manifestPath)}`, `fixtures=${regressionCorpus.fixtures.length}`, ...regressionCorpus.errors.map((error) => `error=${error}`), ); const suites = AUTOMATION_SUITES.filter((candidate) => suiteIsInMode(candidate, options.mode)); const executions = await runSuitesWithConcurrency( suites, options.suiteConcurrency, (suite) => runSuite(suite, outputDir, options.timeoutMs), ); for (let index = 0; index < suites.length; index += 1) { const suite = suites[index]; const execution = executions[index]; for (const scenarioId of suite.scenarios) { const scenario = byId.get(scenarioId); scenario.status = execution.code === 0 && !execution.timedOut ? 'passed' : 'failed'; scenario.reason = scenario.status === 'passed' ? null : 'automation_suite_failed'; scenario.evidence.push( `suite=${suite.id}`, `log=${execution.logPath}`, ...suite.cases[scenarioId].map((assertedCase) => `asserted_case=${assertedCase}`), ); } } const report = createGateReport({ commitSha, branch, artifactSha256: artifact?.sha256 ?? '0'.repeat(64), artifact: artifact ? { path: path.relative(ROOT, options.artifact), kind: artifact.kind, files: artifact.files, bytes: artifact.bytes } : { path: null, kind: 'missing' }, scenarios: results, mode: options.mode, startedAt, completedAt: new Date(), }); await writeGateReport(report, outputDir); return { report, outputDir }; } export async function listChangedPathsBetween(baseCommit, candidateCommit = 'HEAD') { if (!/^[0-9a-f]{40}$/i.test(baseCommit ?? '')) { throw new Error('deployed commit must be a full SHA'); } const exists = await runCommand( 'git', ['cat-file', '-e', `${baseCommit}^{commit}`], { cwd: ROOT, timeoutMs: 30_000 }, ); if (exists.code !== 0) { throw new Error(`deployed commit is not available locally: ${baseCommit}`); } const diff = await runCommand( 'git', ['diff', '--name-only', '-z', `${baseCommit}..${candidateCommit}`], { cwd: ROOT, timeoutMs: 30_000 }, ); if (diff.code !== 0) { throw new Error(`cannot calculate release impact from ${baseCommit} to ${candidateCommit}`); } return diff.stdout .split('\0') .filter(Boolean); } export async function executeImpactReleaseGate(options) { const startedAt = new Date(); assertSafeGateEnvironment({ targets: [options.portalBase] }); assertSafePortalBase(options.portalBase); if (!options.artifact) { throw new Error('impact release gate requires --artifact'); } options.artifact = assertPortalRuntimePath(options.artifact, { repoRoot: ROOT }); if (!options.deployedCommit) { throw new Error('impact release gate requires --deployed-commit'); } const catalog = await loadScenarioCatalog({ root: ROOT }); validateAutomationSuites(catalog); const commitSha = await git('rev-parse', 'HEAD'); const branch = await git('branch', '--show-current'); const outputDir = path.join(options.reportRoot, commitSha); const changedPaths = await listChangedPathsBetween(options.deployedCommit, commitSha); const ancestry = await runCommand( 'git', ['merge-base', '--is-ancestor', options.deployedCommit, commitSha], { cwd: ROOT, timeoutMs: 30_000 }, ); const selection = { ...selectImpactScenarios({ catalog, changedPaths, blockingReasons: ancestry.code === 0 ? [] : [`deployed_commit_not_ancestor:${options.deployedCommit}`], }), base_commit: options.deployedCommit, }; const selectedIds = new Set(selection.selected_ids); const results = catalog .filter((scenario) => selectedIds.has(scenario.id)) .map(scenarioResult); const byId = new Map(results.map((result) => [result.id, result])); const artifact = await applyRepositoryChecks(results, options.artifact); if (selectedIds.has('COMP-09')) { const regressionCorpus = await loadActiveRegressionCorpus({ root: ROOT, catalog }); const comp09 = byId.get('COMP-09'); comp09.reason = regressionCorpus.status === 'ready' ? 'production_regression_replay_not_implemented' : `production_regression_corpus_${regressionCorpus.status}`; comp09.evidence.push( `manifest=${path.relative(ROOT, regressionCorpus.manifestPath)}`, `fixtures=${regressionCorpus.fixtures.length}`, ...regressionCorpus.errors.map((error) => `error=${error}`), ); } const suites = AUTOMATION_SUITES.filter( (suite) => suite.scenarios.some((scenarioId) => selectedIds.has(scenarioId)), ); const executions = await runSuitesWithConcurrency( suites, options.suiteConcurrency, (suite) => runSuite(suite, outputDir, options.timeoutMs), ); for (let index = 0; index < suites.length; index += 1) { const suite = suites[index]; const execution = executions[index]; for (const scenarioId of suite.scenarios) { if (!selectedIds.has(scenarioId)) continue; const scenario = byId.get(scenarioId); scenario.status = execution.code === 0 && !execution.timedOut ? 'passed' : 'failed'; scenario.reason = scenario.status === 'passed' ? null : 'automation_suite_failed'; scenario.evidence.push( `suite=${suite.id}`, `log=${execution.logPath}`, ...suite.cases[scenarioId].map((assertedCase) => `asserted_case=${assertedCase}`), ); } } const completedAt = new Date(); const report = createGateReport({ commitSha, branch, artifactSha256: artifact.sha256, artifact: { path: path.relative(ROOT, options.artifact), kind: artifact.kind, files: artifact.files, bytes: artifact.bytes, }, scenarios: results, mode: 'impact', selection, startedAt, completedAt, }); await writeGateReport(report, outputDir); return { report, outputDir, selection }; } export async function executeIncrementalReleaseGate(options) { const startedAt = new Date(); assertSafeGateEnvironment({ targets: [options.portalBase] }); assertSafePortalBase(options.portalBase); if (!options.artifact) { throw new Error('incremental release gate requires --artifact'); } options.artifact = assertPortalRuntimePath(options.artifact, { repoRoot: ROOT }); const catalog = await loadScenarioCatalog({ root: ROOT }); validateAutomationSuites(catalog); const commitSha = await git('rev-parse', 'HEAD'); const branch = await git('branch', '--show-current'); const outputDir = path.join(options.reportRoot, commitSha); const artifact = await hashArtifact(options.artifact); const baseline = await findCarryForwardBaseline({ reportRoot: options.reportRoot, artifactSha256: artifact.sha256, preferredCommitSha: options.baselineCommit ?? null, deployedCommitSha: options.deployedCommit ?? null, }); if (!baseline) { throw new Error( 'No valid baseline Gate report found for the current artifact; run the full release gate first.', ); } const reexecuted = catalog .filter((scenario) => isReleaseScenario(scenario.id)) .map(scenarioResult); const reexecutedById = new Map(reexecuted.map((result) => [result.id, result])); await applyRepositoryChecks(reexecuted, options.artifact); const upgradeSuites = AUTOMATION_SUITES.filter((suite) => suite.mode === 'upgrade'); const executions = await runSuitesWithConcurrency( upgradeSuites, options.suiteConcurrency, (suite) => runSuite(suite, outputDir, options.timeoutMs), ); for (let index = 0; index < upgradeSuites.length; index += 1) { const suite = upgradeSuites[index]; const execution = executions[index]; for (const scenarioId of suite.scenarios) { if (!isReleaseScenario(scenarioId)) continue; const scenario = reexecutedById.get(scenarioId); scenario.status = execution.code === 0 && !execution.timedOut ? 'passed' : 'failed'; scenario.reason = scenario.status === 'passed' ? null : 'automation_suite_failed'; scenario.evidence.push( `suite=${suite.id}`, `log=${execution.logPath}`, ...suite.cases[scenarioId].map((assertedCase) => `asserted_case=${assertedCase}`), ); } } const mergedScenarios = mergeIncrementalScenarios({ baselineScenarios: baseline.report.scenarios, reexecutedScenarios: reexecuted, }); const completedAt = new Date(); const report = buildIncrementalReport({ baselineReport: baseline.report, baselineReportPath: path.relative(ROOT, baseline.reportPath), commitSha, branch, artifactSha256: artifact.sha256, artifact: { path: path.relative(ROOT, options.artifact), kind: artifact.kind, files: artifact.files, bytes: artifact.bytes, }, scenarios: mergedScenarios, startedAt, completedAt, }); report.summary = summarizeScenarios(mergedScenarios); await writeGateReport(report, outputDir); return { report, outputDir, baseline }; }