Preserve Goose session affinity across canary rollback

This commit is contained in:
john
2026-11-17 14:51:03 +08:00
parent e136f7d3ac
commit ffefdb40cf
7 changed files with 125 additions and 21 deletions
+2 -2
View File
@@ -53,7 +53,7 @@ bash scripts/switch-goose-v149-canary.sh on users
pnpm dev
```
Portal 会注册 `[18006, 18049]` 双 target;canary 用户的新 session 应 pin 到 v1.49(见 `scripts/goose-v149-canary.mjs` + `release-gate/canary-routing.mjs`)。
白名单用户的新会话在候选健康时绑定 v1.49,普通请求的默认池只包含稳定实例。已有会话按已保存的目标继续路由。
## 一键回滚(回落 stable)
@@ -62,7 +62,7 @@ bash scripts/switch-goose-v149-canary.sh rollback
pnpm dev
```
回滚仅移除 canary 块,不删 v1.49 进程或 PG 库;stable `:18006` 保持可用。
回滚后,新会话进入稳定池。保留 v1.49 进程及其 PG 库,让已绑定候选实例的会话继续使用 `TKMIND_GOOSE_CANARY_RESUME_TARGET`。待候选会话退役或完成经过验证的迁移,再运行 `bash scripts/switch-goose-v149-canary.sh off` 清除配置。
## 验证
+2 -1
View File
@@ -22,6 +22,7 @@ import { createExperienceService } from '../experience-service.mjs';
import {
applyGooseV149CanaryBlockEnv,
resolveGooseApiTargetsFromEnv,
resolveGooseCanarySessionTarget,
} from './goose-v149-canary.mjs';
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
@@ -158,7 +159,7 @@ async function bootstrapWorker() {
const apiTarget = apiTargets[0] ?? process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006';
const llmProviderService = createLlmProviderService(pool, {
apiTarget,
apiTargets,
apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(process.env)].filter(Boolean),
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
});
const memoryV2ConfigService = createMemoryV2AdminConfigService(pool);
+14 -3
View File
@@ -208,13 +208,16 @@ export function resolveGooseApiTargetsFromEnv(env = process.env) {
);
}
const targets = [...new Set([...poolTargets, v149])];
const targets = poolTargets.filter((target) => target !== v149);
if (!targets.length) {
throw new Error('Goose users canary requires at least one stable target distinct from the candidate');
}
return Object.freeze({
mode,
stable,
stable: targets[0],
v149,
primary: stable,
primary: targets[0],
targets,
policy,
});
@@ -232,6 +235,14 @@ export function resolveGooseTargetForIdentity(identity, config) {
return config.stable;
}
export function resolveGooseCanarySessionTarget(env, config = resolveGooseApiTargetsFromEnv(env)) {
if (config?.mode === 'users') return config.v149;
const resumeTarget = env.TKMIND_GOOSE_CANARY_RESUME_TARGET?.trim();
return !config && resumeTarget
? assertLoopbackGooseTarget(resumeTarget, 'Goose canary resume target')
: null;
}
export function describeGooseCanaryConfig(env = process.env) {
const config = resolveGooseApiTargetsFromEnv(env);
if (!config) {
+65 -3
View File
@@ -1,5 +1,8 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { createServer } from 'node:http';
import { once } from 'node:events';
import { createTkmindProxy } from '../tkmind-proxy.mjs';
import {
assertLoopbackGooseTarget,
@@ -25,11 +28,11 @@ test('resolveGooseApiTargetsFromEnv all mode pins Portal to v149', () => {
assert.deepEqual(config.targets, ['https://127.0.0.1:18049']);
});
test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', () => {
test('users mode excludes the candidate from the default pool even if listed explicitly', () => {
const config = resolveGooseApiTargetsFromEnv({
TKMIND_GOOSE_CANARY: 'users',
TKMIND_GOOSE_CANARY_USER_IDS: 'user-1',
TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008',
TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008,https://127.0.0.1:18015',
TKMIND_API_TARGET_V149: 'https://127.0.0.1:18015',
});
assert.equal(config.mode, 'users');
@@ -38,10 +41,69 @@ test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', (
'https://127.0.0.1:18006',
'https://127.0.0.1:18007',
'https://127.0.0.1:18008',
'https://127.0.0.1:18015',
]);
});
test('rollback stops new candidate sessions while preserving both session bindings', async (t) => {
async function startUpstream(label) {
const server = createServer((req, res) => {
res.setHeader('content-type', 'application/json');
res.end(JSON.stringify({ id: label }));
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
t.after(() => new Promise((resolve) => server.close(resolve)));
return `http://127.0.0.1:${server.address().port}`;
}
const stable = await startUpstream('stable');
const candidate = await startUpstream('candidate');
const env = {
TKMIND_GOOSE_CANARY: 'users',
TKMIND_GOOSE_CANARY_USER_IDS: 'allowed',
TKMIND_GOOSE_CANARY_USERNAMES: '',
TKMIND_GOOSE_CANARY_WECHAT_USER_IDS: '',
TKMIND_API_TARGET: stable,
TKMIND_API_TARGETS: stable,
TKMIND_API_TARGET_1: '',
TKMIND_API_TARGET_V149: candidate,
TKMIND_GOOSE_CANARY_RESUME_TARGET: '',
MEMIND_RUNTIME_REDIS_URL: '',
};
const originalEnv = Object.fromEntries(Object.keys(env).map((key) => [key, process.env[key]]));
t.after(() => {
for (const [key, value] of Object.entries(originalEnv)) {
if (value === undefined) delete process.env[key];
else process.env[key] = value;
}
});
Object.assign(process.env, env);
const sessions = new Map([['old-session', stable]]);
const sessionAccess = {
async registerSession({ sessionId, target }) { sessions.set(sessionId, target); },
async getSessionTarget(sessionId) { return { target: sessions.get(sessionId), node: 0 }; },
};
const createProxy = (apiTargets) => createTkmindProxy({
apiTargets,
sessionAccess,
userAuth: { async getUserPublishLayout() { return null; } },
});
const options = { workingDir: '/tmp', sessionPolicy: {} };
const active = createProxy(resolveGooseApiTargetsFromEnv().targets);
assert.equal(await active.resolveTarget('old-session'), stable);
assert.equal((await active.startSessionForUser('other', options)).id, 'stable');
assert.equal((await active.startSessionForUser('allowed', options)).id, 'candidate');
assert.equal(await active.resolveTarget('candidate'), candidate);
process.env.TKMIND_GOOSE_CANARY = 'off';
process.env.TKMIND_GOOSE_CANARY_RESUME_TARGET = candidate;
const rolledBack = createProxy([stable]);
assert.equal((await rolledBack.startSessionForUser('allowed', options)).id, 'stable');
assert.equal(await rolledBack.resolveTarget('old-session'), stable);
assert.equal(await rolledBack.resolveTarget('candidate'), candidate);
});
test('resolveGooseTargetForIdentity routes canary users to v149 only', () => {
const config = resolveGooseApiTargetsFromEnv({
TKMIND_GOOSE_CANARY: 'users',
+26 -3
View File
@@ -18,7 +18,8 @@ Usage: bash scripts/switch-goose-v149-canary.sh <command>
Commands:
on [all|users] Enable v1.49 canary (default: all → Portal uses ${V149_TARGET})
off|rollback Restore stable target (${STABLE_TARGET}) and disable canary flags
rollback Stop new canary sessions and retain the candidate for existing sessions
off Remove the canary block after candidate sessions are retired
status Print current canary block / env summary
Env overrides:
@@ -56,7 +57,7 @@ write_canary_block() {
echo "TKMIND_API_TARGETS="
elif [[ "${mode}" == "users" ]]; then
echo "TKMIND_API_TARGET=${STABLE_TARGET}"
echo "TKMIND_API_TARGETS=${STABLE_TARGET},${V149_TARGET}"
echo "TKMIND_API_TARGETS=${STABLE_TARGET}"
if [[ -n "${TKMIND_GOOSE_CANARY_USER_IDS:-}" ]]; then
echo "TKMIND_GOOSE_CANARY_USER_IDS=${TKMIND_GOOSE_CANARY_USER_IDS}"
fi
@@ -95,7 +96,29 @@ case "${cmd}" in
echo "[goose-v149-canary] ensure v1.49 is running: bash scripts/run-goosed-v149-local.sh"
echo "[goose-v149-canary] verify: curl -sk ${V149_TARGET}/status"
;;
off|rollback)
rollback)
if [[ -f "${ENV_FILE}" ]]; then
saved_target="$(awk -v begin="${MARKER_BEGIN}" -v end="${MARKER_END}" '
$0 == begin { active=1; next }
$0 == end { exit }
active && /^TKMIND_(API_TARGET_V149|GOOSE_CANARY_RESUME_TARGET)=/ {
sub(/^[^=]*=/, "")
print
exit
}
' "${ENV_FILE}")"
V149_TARGET="${saved_target:-${V149_TARGET}}"
fi
remove_canary_block
{
echo "${MARKER_BEGIN}"
echo "TKMIND_GOOSE_CANARY=off"
echo "TKMIND_GOOSE_CANARY_RESUME_TARGET=${V149_TARGET}"
echo "${MARKER_END}"
} >> "${ENV_FILE}"
echo "[goose-v149-canary] candidate kept for existing sessions; keep its process running"
;;
off)
remove_canary_block
echo "[goose-v149-canary] disabled; restored env block removed from ${ENV_FILE}"
echo "[goose-v149-canary] Portal falls back to default TKMIND_API_TARGET (${STABLE_TARGET}) from .env"
+3 -2
View File
@@ -4,6 +4,7 @@ import { createExperienceService } from '../experience-service.mjs';
import { createImageMakeAdminConfigService } from '../image-make-admin-config.mjs';
import { createImageMakeClientFromEnv } from '../image-make-client.mjs';
import { createLlmProviderService, RELAY_BOOTSTRAP } from '../llm-providers.mjs';
import { resolveGooseCanarySessionTarget } from '../scripts/goose-v149-canary.mjs';
import { createMindSpaceAgentRunner } from '../mindspace-agent-runner.mjs';
import { createMindSpaceAuditWriter } from '../mindspace-audit.mjs';
import { createMindSpaceImageGenerationService } from '../mindspace-image-generation.mjs';
@@ -23,7 +24,7 @@ export async function bootstrapPortalAgentServices({
env = process.env,
runtime,
apiTarget,
apiTargets,
apiTargets = [],
apiSecret,
userAuth,
sessionAccess,
@@ -162,7 +163,7 @@ export async function bootstrapPortalAgentServices({
const llmProviderService =
createLlmProviderServiceFn(pool, {
apiTarget,
apiTargets,
apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(env)].filter(Boolean),
apiSecret,
});
const assetGatewayConfigService =
+13 -7
View File
@@ -28,6 +28,7 @@ import { createSessionAccess, isSessionBrokerEnabled } from './session-broker.mj
import { createSessionBrokerMetrics, isSessionBrokerMetricsEnabled } from './session-broker-metrics.mjs';
import {
resolveGooseApiTargetsFromEnv,
resolveGooseCanarySessionTarget,
resolveGooseTargetForIdentity,
} from './scripts/goose-v149-canary.mjs';
import { createImgproxySigner } from './imgproxy-signer.mjs';
@@ -1194,17 +1195,22 @@ export function createTkmindProxy({
const brokerMetrics = isSessionBrokerMetricsEnabled() && sessionStore.enabled
? createSessionBrokerMetrics({ logger: console })
: null;
const targets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : [];
const primaryTarget = targets[0] ?? apiTarget ?? '';
const configuredTargets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : [];
let gooseCanaryConfig = null;
let resumeTarget = null;
try {
gooseCanaryConfig = resolveGooseApiTargetsFromEnv(process.env);
resumeTarget = resolveGooseCanarySessionTarget(process.env, gooseCanaryConfig);
} catch (err) {
console.warn(
'[TKMindProxy] Goose v1.49 canary config ignored:',
err instanceof Error ? err.message : err,
);
}
// Keep the candidate addressable for pinned sessions, outside the default new-work pool.
const targets = configuredTargets.filter((target) => target !== resumeTarget);
const primaryTarget = targets[0] ?? '';
const sessionTargets = resumeTarget ? [...targets, resumeTarget] : targets;
let rrIdx = 0;
let imgproxySigner = null;
@@ -1231,7 +1237,7 @@ export function createTkmindProxy({
return false;
}
}
const runtimeRouter = createRuntimeRouter({ targets, targetHealthy });
const runtimeRouter = createRuntimeRouter({ targets: sessionTargets, targetHealthy });
function isGeneratedSessionName(name) {
const normalized = typeof name === 'string' ? name.trim() : '';
@@ -1422,7 +1428,7 @@ export function createTkmindProxy({
async function getRuntimeStatus() {
const targetStatuses = [];
for (const target of targets) {
for (const target of sessionTargets) {
targetStatuses.push({
target,
healthy: await targetHealthy(target),
@@ -1608,7 +1614,7 @@ export function createTkmindProxy({
}
async function resolveTarget(sessionId) {
if (targets.length <= 1 || !sessionId) return primaryTarget;
if (sessionTargets.length <= 1 || !sessionId) return primaryTarget;
try {
const routedTarget = await runtimeRouter?.resolveSessionTarget(sessionId);
if (routedTarget) return routedTarget;
@@ -1616,9 +1622,9 @@ export function createTkmindProxy({
// Prefer the pinned URL: stable across reordering/resizing the target list.
// Only honor it if that upstream is still configured; otherwise fall back to
// the legacy integer index, then to primary.
if (target && targets.includes(target)) return target;
if (target && sessionTargets.includes(target)) return target;
if (brokerMetrics) {
if (target && !targets.includes(target)) {
if (target) {
brokerMetrics.resolveTargetMiss({
sessionId,
target,