From ffefdb40cffceaf0289f96dabf328e0e09185b30 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 17 Nov 2026 14:51:03 +0800 Subject: [PATCH] Preserve Goose session affinity across canary rollback --- docs/goose-v149-canary-runbook.md | 4 +- scripts/agent-run-worker.mjs | 3 +- scripts/goose-v149-canary.mjs | 17 +++++- scripts/goose-v149-canary.test.mjs | 68 +++++++++++++++++++++- scripts/switch-goose-v149-canary.sh | 29 ++++++++- server/portal-agent-services-bootstrap.mjs | 5 +- tkmind-proxy.mjs | 20 ++++--- 7 files changed, 125 insertions(+), 21 deletions(-) diff --git a/docs/goose-v149-canary-runbook.md b/docs/goose-v149-canary-runbook.md index fe8a31a..3275354 100644 --- a/docs/goose-v149-canary-runbook.md +++ b/docs/goose-v149-canary-runbook.md @@ -53,7 +53,7 @@ bash scripts/switch-goose-v149-canary.sh on users pnpm dev ``` -Portal 会注册 `[18006, 18049]` 双 target;canary 用户的新 session 应 pin 到 v1.49(见 `scripts/goose-v149-canary.mjs` + `release-gate/canary-routing.mjs`)。 +白名单用户的新会话在候选健康时绑定 v1.49,普通请求的默认池只包含稳定实例。已有会话按已保存的目标继续路由。 ## 一键回滚(回落 stable) @@ -62,7 +62,7 @@ bash scripts/switch-goose-v149-canary.sh rollback pnpm dev ``` -回滚仅移除 canary 块,不删 v1.49 进程或 PG 库;stable `:18006` 保持可用。 +回滚后,新会话进入稳定池。保留 v1.49 进程及其 PG 库,让已绑定候选实例的会话继续使用 `TKMIND_GOOSE_CANARY_RESUME_TARGET`。待候选会话退役或完成经过验证的迁移,再运行 `bash scripts/switch-goose-v149-canary.sh off` 清除配置。 ## 验证 diff --git a/scripts/agent-run-worker.mjs b/scripts/agent-run-worker.mjs index 6d1148d..29d730f 100644 --- a/scripts/agent-run-worker.mjs +++ b/scripts/agent-run-worker.mjs @@ -22,6 +22,7 @@ import { createExperienceService } from '../experience-service.mjs'; import { applyGooseV149CanaryBlockEnv, resolveGooseApiTargetsFromEnv, + resolveGooseCanarySessionTarget, } from './goose-v149-canary.mjs'; import { loadMemindEnvFiles } from './memind-runtime-profile.mjs'; @@ -158,7 +159,7 @@ async function bootstrapWorker() { const apiTarget = apiTargets[0] ?? process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006'; const llmProviderService = createLlmProviderService(pool, { apiTarget, - apiTargets, + apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(process.env)].filter(Boolean), apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret', }); const memoryV2ConfigService = createMemoryV2AdminConfigService(pool); diff --git a/scripts/goose-v149-canary.mjs b/scripts/goose-v149-canary.mjs index d16bde6..e61f553 100644 --- a/scripts/goose-v149-canary.mjs +++ b/scripts/goose-v149-canary.mjs @@ -208,13 +208,16 @@ export function resolveGooseApiTargetsFromEnv(env = process.env) { ); } - const targets = [...new Set([...poolTargets, v149])]; + const targets = poolTargets.filter((target) => target !== v149); + if (!targets.length) { + throw new Error('Goose users canary requires at least one stable target distinct from the candidate'); + } return Object.freeze({ mode, - stable, + stable: targets[0], v149, - primary: stable, + primary: targets[0], targets, policy, }); @@ -232,6 +235,14 @@ export function resolveGooseTargetForIdentity(identity, config) { return config.stable; } +export function resolveGooseCanarySessionTarget(env, config = resolveGooseApiTargetsFromEnv(env)) { + if (config?.mode === 'users') return config.v149; + const resumeTarget = env.TKMIND_GOOSE_CANARY_RESUME_TARGET?.trim(); + return !config && resumeTarget + ? assertLoopbackGooseTarget(resumeTarget, 'Goose canary resume target') + : null; +} + export function describeGooseCanaryConfig(env = process.env) { const config = resolveGooseApiTargetsFromEnv(env); if (!config) { diff --git a/scripts/goose-v149-canary.test.mjs b/scripts/goose-v149-canary.test.mjs index 86ee80f..03399a1 100644 --- a/scripts/goose-v149-canary.test.mjs +++ b/scripts/goose-v149-canary.test.mjs @@ -1,5 +1,8 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { createTkmindProxy } from '../tkmind-proxy.mjs'; import { assertLoopbackGooseTarget, @@ -25,11 +28,11 @@ test('resolveGooseApiTargetsFromEnv all mode pins Portal to v149', () => { assert.deepEqual(config.targets, ['https://127.0.0.1:18049']); }); -test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', () => { +test('users mode excludes the candidate from the default pool even if listed explicitly', () => { const config = resolveGooseApiTargetsFromEnv({ TKMIND_GOOSE_CANARY: 'users', TKMIND_GOOSE_CANARY_USER_IDS: 'user-1', - TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008', + TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008,https://127.0.0.1:18015', TKMIND_API_TARGET_V149: 'https://127.0.0.1:18015', }); assert.equal(config.mode, 'users'); @@ -38,10 +41,69 @@ test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', ( 'https://127.0.0.1:18006', 'https://127.0.0.1:18007', 'https://127.0.0.1:18008', - 'https://127.0.0.1:18015', ]); }); +test('rollback stops new candidate sessions while preserving both session bindings', async (t) => { + async function startUpstream(label) { + const server = createServer((req, res) => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({ id: label })); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + t.after(() => new Promise((resolve) => server.close(resolve))); + return `http://127.0.0.1:${server.address().port}`; + } + + const stable = await startUpstream('stable'); + const candidate = await startUpstream('candidate'); + const env = { + TKMIND_GOOSE_CANARY: 'users', + TKMIND_GOOSE_CANARY_USER_IDS: 'allowed', + TKMIND_GOOSE_CANARY_USERNAMES: '', + TKMIND_GOOSE_CANARY_WECHAT_USER_IDS: '', + TKMIND_API_TARGET: stable, + TKMIND_API_TARGETS: stable, + TKMIND_API_TARGET_1: '', + TKMIND_API_TARGET_V149: candidate, + TKMIND_GOOSE_CANARY_RESUME_TARGET: '', + MEMIND_RUNTIME_REDIS_URL: '', + }; + const originalEnv = Object.fromEntries(Object.keys(env).map((key) => [key, process.env[key]])); + t.after(() => { + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + }); + Object.assign(process.env, env); + + const sessions = new Map([['old-session', stable]]); + const sessionAccess = { + async registerSession({ sessionId, target }) { sessions.set(sessionId, target); }, + async getSessionTarget(sessionId) { return { target: sessions.get(sessionId), node: 0 }; }, + }; + const createProxy = (apiTargets) => createTkmindProxy({ + apiTargets, + sessionAccess, + userAuth: { async getUserPublishLayout() { return null; } }, + }); + const options = { workingDir: '/tmp', sessionPolicy: {} }; + const active = createProxy(resolveGooseApiTargetsFromEnv().targets); + assert.equal(await active.resolveTarget('old-session'), stable); + assert.equal((await active.startSessionForUser('other', options)).id, 'stable'); + assert.equal((await active.startSessionForUser('allowed', options)).id, 'candidate'); + assert.equal(await active.resolveTarget('candidate'), candidate); + + process.env.TKMIND_GOOSE_CANARY = 'off'; + process.env.TKMIND_GOOSE_CANARY_RESUME_TARGET = candidate; + const rolledBack = createProxy([stable]); + assert.equal((await rolledBack.startSessionForUser('allowed', options)).id, 'stable'); + assert.equal(await rolledBack.resolveTarget('old-session'), stable); + assert.equal(await rolledBack.resolveTarget('candidate'), candidate); +}); + test('resolveGooseTargetForIdentity routes canary users to v149 only', () => { const config = resolveGooseApiTargetsFromEnv({ TKMIND_GOOSE_CANARY: 'users', diff --git a/scripts/switch-goose-v149-canary.sh b/scripts/switch-goose-v149-canary.sh index c92a761..7c781dc 100755 --- a/scripts/switch-goose-v149-canary.sh +++ b/scripts/switch-goose-v149-canary.sh @@ -18,7 +18,8 @@ Usage: bash scripts/switch-goose-v149-canary.sh Commands: on [all|users] Enable v1.49 canary (default: all → Portal uses ${V149_TARGET}) - off|rollback Restore stable target (${STABLE_TARGET}) and disable canary flags + rollback Stop new canary sessions and retain the candidate for existing sessions + off Remove the canary block after candidate sessions are retired status Print current canary block / env summary Env overrides: @@ -56,7 +57,7 @@ write_canary_block() { echo "TKMIND_API_TARGETS=" elif [[ "${mode}" == "users" ]]; then echo "TKMIND_API_TARGET=${STABLE_TARGET}" - echo "TKMIND_API_TARGETS=${STABLE_TARGET},${V149_TARGET}" + echo "TKMIND_API_TARGETS=${STABLE_TARGET}" if [[ -n "${TKMIND_GOOSE_CANARY_USER_IDS:-}" ]]; then echo "TKMIND_GOOSE_CANARY_USER_IDS=${TKMIND_GOOSE_CANARY_USER_IDS}" fi @@ -95,7 +96,29 @@ case "${cmd}" in echo "[goose-v149-canary] ensure v1.49 is running: bash scripts/run-goosed-v149-local.sh" echo "[goose-v149-canary] verify: curl -sk ${V149_TARGET}/status" ;; - off|rollback) + rollback) + if [[ -f "${ENV_FILE}" ]]; then + saved_target="$(awk -v begin="${MARKER_BEGIN}" -v end="${MARKER_END}" ' + $0 == begin { active=1; next } + $0 == end { exit } + active && /^TKMIND_(API_TARGET_V149|GOOSE_CANARY_RESUME_TARGET)=/ { + sub(/^[^=]*=/, "") + print + exit + } + ' "${ENV_FILE}")" + V149_TARGET="${saved_target:-${V149_TARGET}}" + fi + remove_canary_block + { + echo "${MARKER_BEGIN}" + echo "TKMIND_GOOSE_CANARY=off" + echo "TKMIND_GOOSE_CANARY_RESUME_TARGET=${V149_TARGET}" + echo "${MARKER_END}" + } >> "${ENV_FILE}" + echo "[goose-v149-canary] candidate kept for existing sessions; keep its process running" + ;; + off) remove_canary_block echo "[goose-v149-canary] disabled; restored env block removed from ${ENV_FILE}" echo "[goose-v149-canary] Portal falls back to default TKMIND_API_TARGET (${STABLE_TARGET}) from .env" diff --git a/server/portal-agent-services-bootstrap.mjs b/server/portal-agent-services-bootstrap.mjs index 4f6ddf6..7b2aa15 100644 --- a/server/portal-agent-services-bootstrap.mjs +++ b/server/portal-agent-services-bootstrap.mjs @@ -4,6 +4,7 @@ import { createExperienceService } from '../experience-service.mjs'; import { createImageMakeAdminConfigService } from '../image-make-admin-config.mjs'; import { createImageMakeClientFromEnv } from '../image-make-client.mjs'; import { createLlmProviderService, RELAY_BOOTSTRAP } from '../llm-providers.mjs'; +import { resolveGooseCanarySessionTarget } from '../scripts/goose-v149-canary.mjs'; import { createMindSpaceAgentRunner } from '../mindspace-agent-runner.mjs'; import { createMindSpaceAuditWriter } from '../mindspace-audit.mjs'; import { createMindSpaceImageGenerationService } from '../mindspace-image-generation.mjs'; @@ -23,7 +24,7 @@ export async function bootstrapPortalAgentServices({ env = process.env, runtime, apiTarget, - apiTargets, + apiTargets = [], apiSecret, userAuth, sessionAccess, @@ -162,7 +163,7 @@ export async function bootstrapPortalAgentServices({ const llmProviderService = createLlmProviderServiceFn(pool, { apiTarget, - apiTargets, + apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(env)].filter(Boolean), apiSecret, }); const assetGatewayConfigService = diff --git a/tkmind-proxy.mjs b/tkmind-proxy.mjs index 2272039..9e19ee9 100644 --- a/tkmind-proxy.mjs +++ b/tkmind-proxy.mjs @@ -28,6 +28,7 @@ import { createSessionAccess, isSessionBrokerEnabled } from './session-broker.mj import { createSessionBrokerMetrics, isSessionBrokerMetricsEnabled } from './session-broker-metrics.mjs'; import { resolveGooseApiTargetsFromEnv, + resolveGooseCanarySessionTarget, resolveGooseTargetForIdentity, } from './scripts/goose-v149-canary.mjs'; import { createImgproxySigner } from './imgproxy-signer.mjs'; @@ -1194,17 +1195,22 @@ export function createTkmindProxy({ const brokerMetrics = isSessionBrokerMetricsEnabled() && sessionStore.enabled ? createSessionBrokerMetrics({ logger: console }) : null; - const targets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : []; - const primaryTarget = targets[0] ?? apiTarget ?? ''; + const configuredTargets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : []; let gooseCanaryConfig = null; + let resumeTarget = null; try { gooseCanaryConfig = resolveGooseApiTargetsFromEnv(process.env); + resumeTarget = resolveGooseCanarySessionTarget(process.env, gooseCanaryConfig); } catch (err) { console.warn( '[TKMindProxy] Goose v1.49 canary config ignored:', err instanceof Error ? err.message : err, ); } + // Keep the candidate addressable for pinned sessions, outside the default new-work pool. + const targets = configuredTargets.filter((target) => target !== resumeTarget); + const primaryTarget = targets[0] ?? ''; + const sessionTargets = resumeTarget ? [...targets, resumeTarget] : targets; let rrIdx = 0; let imgproxySigner = null; @@ -1231,7 +1237,7 @@ export function createTkmindProxy({ return false; } } - const runtimeRouter = createRuntimeRouter({ targets, targetHealthy }); + const runtimeRouter = createRuntimeRouter({ targets: sessionTargets, targetHealthy }); function isGeneratedSessionName(name) { const normalized = typeof name === 'string' ? name.trim() : ''; @@ -1422,7 +1428,7 @@ export function createTkmindProxy({ async function getRuntimeStatus() { const targetStatuses = []; - for (const target of targets) { + for (const target of sessionTargets) { targetStatuses.push({ target, healthy: await targetHealthy(target), @@ -1608,7 +1614,7 @@ export function createTkmindProxy({ } async function resolveTarget(sessionId) { - if (targets.length <= 1 || !sessionId) return primaryTarget; + if (sessionTargets.length <= 1 || !sessionId) return primaryTarget; try { const routedTarget = await runtimeRouter?.resolveSessionTarget(sessionId); if (routedTarget) return routedTarget; @@ -1616,9 +1622,9 @@ export function createTkmindProxy({ // Prefer the pinned URL: stable across reordering/resizing the target list. // Only honor it if that upstream is still configured; otherwise fall back to // the legacy integer index, then to primary. - if (target && targets.includes(target)) return target; + if (target && sessionTargets.includes(target)) return target; if (brokerMetrics) { - if (target && !targets.includes(target)) { + if (target) { brokerMetrics.resolveTargetMiss({ sessionId, target,