Preserve Goose session affinity across canary rollback
This commit is contained in:
@@ -53,7 +53,7 @@ bash scripts/switch-goose-v149-canary.sh on users
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
Portal 会注册 `[18006, 18049]` 双 target;canary 用户的新 session 应 pin 到 v1.49(见 `scripts/goose-v149-canary.mjs` + `release-gate/canary-routing.mjs`)。
|
||||
白名单用户的新会话在候选健康时绑定 v1.49,普通请求的默认池只包含稳定实例。已有会话按已保存的目标继续路由。
|
||||
|
||||
## 一键回滚(回落 stable)
|
||||
|
||||
@@ -62,7 +62,7 @@ bash scripts/switch-goose-v149-canary.sh rollback
|
||||
pnpm dev
|
||||
```
|
||||
|
||||
回滚仅移除 canary 块,不删 v1.49 进程或 PG 库;stable `:18006` 保持可用。
|
||||
回滚后,新会话进入稳定池。保留 v1.49 进程及其 PG 库,让已绑定候选实例的会话继续使用 `TKMIND_GOOSE_CANARY_RESUME_TARGET`。待候选会话退役或完成经过验证的迁移,再运行 `bash scripts/switch-goose-v149-canary.sh off` 清除配置。
|
||||
|
||||
## 验证
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ import { createExperienceService } from '../experience-service.mjs';
|
||||
import {
|
||||
applyGooseV149CanaryBlockEnv,
|
||||
resolveGooseApiTargetsFromEnv,
|
||||
resolveGooseCanarySessionTarget,
|
||||
} from './goose-v149-canary.mjs';
|
||||
import { loadMemindEnvFiles } from './memind-runtime-profile.mjs';
|
||||
|
||||
@@ -158,7 +159,7 @@ async function bootstrapWorker() {
|
||||
const apiTarget = apiTargets[0] ?? process.env.TKMIND_API_TARGET ?? 'https://127.0.0.1:18006';
|
||||
const llmProviderService = createLlmProviderService(pool, {
|
||||
apiTarget,
|
||||
apiTargets,
|
||||
apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(process.env)].filter(Boolean),
|
||||
apiSecret: process.env.TKMIND_SERVER__SECRET_KEY ?? 'local-dev-secret',
|
||||
});
|
||||
const memoryV2ConfigService = createMemoryV2AdminConfigService(pool);
|
||||
|
||||
@@ -208,13 +208,16 @@ export function resolveGooseApiTargetsFromEnv(env = process.env) {
|
||||
);
|
||||
}
|
||||
|
||||
const targets = [...new Set([...poolTargets, v149])];
|
||||
const targets = poolTargets.filter((target) => target !== v149);
|
||||
if (!targets.length) {
|
||||
throw new Error('Goose users canary requires at least one stable target distinct from the candidate');
|
||||
}
|
||||
|
||||
return Object.freeze({
|
||||
mode,
|
||||
stable,
|
||||
stable: targets[0],
|
||||
v149,
|
||||
primary: stable,
|
||||
primary: targets[0],
|
||||
targets,
|
||||
policy,
|
||||
});
|
||||
@@ -232,6 +235,14 @@ export function resolveGooseTargetForIdentity(identity, config) {
|
||||
return config.stable;
|
||||
}
|
||||
|
||||
export function resolveGooseCanarySessionTarget(env, config = resolveGooseApiTargetsFromEnv(env)) {
|
||||
if (config?.mode === 'users') return config.v149;
|
||||
const resumeTarget = env.TKMIND_GOOSE_CANARY_RESUME_TARGET?.trim();
|
||||
return !config && resumeTarget
|
||||
? assertLoopbackGooseTarget(resumeTarget, 'Goose canary resume target')
|
||||
: null;
|
||||
}
|
||||
|
||||
export function describeGooseCanaryConfig(env = process.env) {
|
||||
const config = resolveGooseApiTargetsFromEnv(env);
|
||||
if (!config) {
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:http';
|
||||
import { once } from 'node:events';
|
||||
import { createTkmindProxy } from '../tkmind-proxy.mjs';
|
||||
|
||||
import {
|
||||
assertLoopbackGooseTarget,
|
||||
@@ -25,11 +28,11 @@ test('resolveGooseApiTargetsFromEnv all mode pins Portal to v149', () => {
|
||||
assert.deepEqual(config.targets, ['https://127.0.0.1:18049']);
|
||||
});
|
||||
|
||||
test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', () => {
|
||||
test('users mode excludes the candidate from the default pool even if listed explicitly', () => {
|
||||
const config = resolveGooseApiTargetsFromEnv({
|
||||
TKMIND_GOOSE_CANARY: 'users',
|
||||
TKMIND_GOOSE_CANARY_USER_IDS: 'user-1',
|
||||
TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008',
|
||||
TKMIND_API_TARGETS: 'https://127.0.0.1:18006,https://127.0.0.1:18007,https://127.0.0.1:18008,https://127.0.0.1:18015',
|
||||
TKMIND_API_TARGET_V149: 'https://127.0.0.1:18015',
|
||||
});
|
||||
assert.equal(config.mode, 'users');
|
||||
@@ -38,10 +41,69 @@ test('resolveGooseApiTargetsFromEnv users mode keeps full pool and adds v149', (
|
||||
'https://127.0.0.1:18006',
|
||||
'https://127.0.0.1:18007',
|
||||
'https://127.0.0.1:18008',
|
||||
'https://127.0.0.1:18015',
|
||||
]);
|
||||
});
|
||||
|
||||
test('rollback stops new candidate sessions while preserving both session bindings', async (t) => {
|
||||
async function startUpstream(label) {
|
||||
const server = createServer((req, res) => {
|
||||
res.setHeader('content-type', 'application/json');
|
||||
res.end(JSON.stringify({ id: label }));
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
t.after(() => new Promise((resolve) => server.close(resolve)));
|
||||
return `http://127.0.0.1:${server.address().port}`;
|
||||
}
|
||||
|
||||
const stable = await startUpstream('stable');
|
||||
const candidate = await startUpstream('candidate');
|
||||
const env = {
|
||||
TKMIND_GOOSE_CANARY: 'users',
|
||||
TKMIND_GOOSE_CANARY_USER_IDS: 'allowed',
|
||||
TKMIND_GOOSE_CANARY_USERNAMES: '',
|
||||
TKMIND_GOOSE_CANARY_WECHAT_USER_IDS: '',
|
||||
TKMIND_API_TARGET: stable,
|
||||
TKMIND_API_TARGETS: stable,
|
||||
TKMIND_API_TARGET_1: '',
|
||||
TKMIND_API_TARGET_V149: candidate,
|
||||
TKMIND_GOOSE_CANARY_RESUME_TARGET: '',
|
||||
MEMIND_RUNTIME_REDIS_URL: '',
|
||||
};
|
||||
const originalEnv = Object.fromEntries(Object.keys(env).map((key) => [key, process.env[key]]));
|
||||
t.after(() => {
|
||||
for (const [key, value] of Object.entries(originalEnv)) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
});
|
||||
Object.assign(process.env, env);
|
||||
|
||||
const sessions = new Map([['old-session', stable]]);
|
||||
const sessionAccess = {
|
||||
async registerSession({ sessionId, target }) { sessions.set(sessionId, target); },
|
||||
async getSessionTarget(sessionId) { return { target: sessions.get(sessionId), node: 0 }; },
|
||||
};
|
||||
const createProxy = (apiTargets) => createTkmindProxy({
|
||||
apiTargets,
|
||||
sessionAccess,
|
||||
userAuth: { async getUserPublishLayout() { return null; } },
|
||||
});
|
||||
const options = { workingDir: '/tmp', sessionPolicy: {} };
|
||||
const active = createProxy(resolveGooseApiTargetsFromEnv().targets);
|
||||
assert.equal(await active.resolveTarget('old-session'), stable);
|
||||
assert.equal((await active.startSessionForUser('other', options)).id, 'stable');
|
||||
assert.equal((await active.startSessionForUser('allowed', options)).id, 'candidate');
|
||||
assert.equal(await active.resolveTarget('candidate'), candidate);
|
||||
|
||||
process.env.TKMIND_GOOSE_CANARY = 'off';
|
||||
process.env.TKMIND_GOOSE_CANARY_RESUME_TARGET = candidate;
|
||||
const rolledBack = createProxy([stable]);
|
||||
assert.equal((await rolledBack.startSessionForUser('allowed', options)).id, 'stable');
|
||||
assert.equal(await rolledBack.resolveTarget('old-session'), stable);
|
||||
assert.equal(await rolledBack.resolveTarget('candidate'), candidate);
|
||||
});
|
||||
|
||||
test('resolveGooseTargetForIdentity routes canary users to v149 only', () => {
|
||||
const config = resolveGooseApiTargetsFromEnv({
|
||||
TKMIND_GOOSE_CANARY: 'users',
|
||||
|
||||
@@ -18,7 +18,8 @@ Usage: bash scripts/switch-goose-v149-canary.sh <command>
|
||||
|
||||
Commands:
|
||||
on [all|users] Enable v1.49 canary (default: all → Portal uses ${V149_TARGET})
|
||||
off|rollback Restore stable target (${STABLE_TARGET}) and disable canary flags
|
||||
rollback Stop new canary sessions and retain the candidate for existing sessions
|
||||
off Remove the canary block after candidate sessions are retired
|
||||
status Print current canary block / env summary
|
||||
|
||||
Env overrides:
|
||||
@@ -56,7 +57,7 @@ write_canary_block() {
|
||||
echo "TKMIND_API_TARGETS="
|
||||
elif [[ "${mode}" == "users" ]]; then
|
||||
echo "TKMIND_API_TARGET=${STABLE_TARGET}"
|
||||
echo "TKMIND_API_TARGETS=${STABLE_TARGET},${V149_TARGET}"
|
||||
echo "TKMIND_API_TARGETS=${STABLE_TARGET}"
|
||||
if [[ -n "${TKMIND_GOOSE_CANARY_USER_IDS:-}" ]]; then
|
||||
echo "TKMIND_GOOSE_CANARY_USER_IDS=${TKMIND_GOOSE_CANARY_USER_IDS}"
|
||||
fi
|
||||
@@ -95,7 +96,29 @@ case "${cmd}" in
|
||||
echo "[goose-v149-canary] ensure v1.49 is running: bash scripts/run-goosed-v149-local.sh"
|
||||
echo "[goose-v149-canary] verify: curl -sk ${V149_TARGET}/status"
|
||||
;;
|
||||
off|rollback)
|
||||
rollback)
|
||||
if [[ -f "${ENV_FILE}" ]]; then
|
||||
saved_target="$(awk -v begin="${MARKER_BEGIN}" -v end="${MARKER_END}" '
|
||||
$0 == begin { active=1; next }
|
||||
$0 == end { exit }
|
||||
active && /^TKMIND_(API_TARGET_V149|GOOSE_CANARY_RESUME_TARGET)=/ {
|
||||
sub(/^[^=]*=/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' "${ENV_FILE}")"
|
||||
V149_TARGET="${saved_target:-${V149_TARGET}}"
|
||||
fi
|
||||
remove_canary_block
|
||||
{
|
||||
echo "${MARKER_BEGIN}"
|
||||
echo "TKMIND_GOOSE_CANARY=off"
|
||||
echo "TKMIND_GOOSE_CANARY_RESUME_TARGET=${V149_TARGET}"
|
||||
echo "${MARKER_END}"
|
||||
} >> "${ENV_FILE}"
|
||||
echo "[goose-v149-canary] candidate kept for existing sessions; keep its process running"
|
||||
;;
|
||||
off)
|
||||
remove_canary_block
|
||||
echo "[goose-v149-canary] disabled; restored env block removed from ${ENV_FILE}"
|
||||
echo "[goose-v149-canary] Portal falls back to default TKMIND_API_TARGET (${STABLE_TARGET}) from .env"
|
||||
|
||||
@@ -4,6 +4,7 @@ import { createExperienceService } from '../experience-service.mjs';
|
||||
import { createImageMakeAdminConfigService } from '../image-make-admin-config.mjs';
|
||||
import { createImageMakeClientFromEnv } from '../image-make-client.mjs';
|
||||
import { createLlmProviderService, RELAY_BOOTSTRAP } from '../llm-providers.mjs';
|
||||
import { resolveGooseCanarySessionTarget } from '../scripts/goose-v149-canary.mjs';
|
||||
import { createMindSpaceAgentRunner } from '../mindspace-agent-runner.mjs';
|
||||
import { createMindSpaceAuditWriter } from '../mindspace-audit.mjs';
|
||||
import { createMindSpaceImageGenerationService } from '../mindspace-image-generation.mjs';
|
||||
@@ -23,7 +24,7 @@ export async function bootstrapPortalAgentServices({
|
||||
env = process.env,
|
||||
runtime,
|
||||
apiTarget,
|
||||
apiTargets,
|
||||
apiTargets = [],
|
||||
apiSecret,
|
||||
userAuth,
|
||||
sessionAccess,
|
||||
@@ -162,7 +163,7 @@ export async function bootstrapPortalAgentServices({
|
||||
const llmProviderService =
|
||||
createLlmProviderServiceFn(pool, {
|
||||
apiTarget,
|
||||
apiTargets,
|
||||
apiTargets: [...apiTargets, resolveGooseCanarySessionTarget(env)].filter(Boolean),
|
||||
apiSecret,
|
||||
});
|
||||
const assetGatewayConfigService =
|
||||
|
||||
+13
-7
@@ -28,6 +28,7 @@ import { createSessionAccess, isSessionBrokerEnabled } from './session-broker.mj
|
||||
import { createSessionBrokerMetrics, isSessionBrokerMetricsEnabled } from './session-broker-metrics.mjs';
|
||||
import {
|
||||
resolveGooseApiTargetsFromEnv,
|
||||
resolveGooseCanarySessionTarget,
|
||||
resolveGooseTargetForIdentity,
|
||||
} from './scripts/goose-v149-canary.mjs';
|
||||
import { createImgproxySigner } from './imgproxy-signer.mjs';
|
||||
@@ -1194,17 +1195,22 @@ export function createTkmindProxy({
|
||||
const brokerMetrics = isSessionBrokerMetricsEnabled() && sessionStore.enabled
|
||||
? createSessionBrokerMetrics({ logger: console })
|
||||
: null;
|
||||
const targets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : [];
|
||||
const primaryTarget = targets[0] ?? apiTarget ?? '';
|
||||
const configuredTargets = apiTargets?.length ? apiTargets : apiTarget ? [apiTarget] : [];
|
||||
let gooseCanaryConfig = null;
|
||||
let resumeTarget = null;
|
||||
try {
|
||||
gooseCanaryConfig = resolveGooseApiTargetsFromEnv(process.env);
|
||||
resumeTarget = resolveGooseCanarySessionTarget(process.env, gooseCanaryConfig);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
'[TKMindProxy] Goose v1.49 canary config ignored:',
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
}
|
||||
// Keep the candidate addressable for pinned sessions, outside the default new-work pool.
|
||||
const targets = configuredTargets.filter((target) => target !== resumeTarget);
|
||||
const primaryTarget = targets[0] ?? '';
|
||||
const sessionTargets = resumeTarget ? [...targets, resumeTarget] : targets;
|
||||
let rrIdx = 0;
|
||||
|
||||
let imgproxySigner = null;
|
||||
@@ -1231,7 +1237,7 @@ export function createTkmindProxy({
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const runtimeRouter = createRuntimeRouter({ targets, targetHealthy });
|
||||
const runtimeRouter = createRuntimeRouter({ targets: sessionTargets, targetHealthy });
|
||||
|
||||
function isGeneratedSessionName(name) {
|
||||
const normalized = typeof name === 'string' ? name.trim() : '';
|
||||
@@ -1422,7 +1428,7 @@ export function createTkmindProxy({
|
||||
|
||||
async function getRuntimeStatus() {
|
||||
const targetStatuses = [];
|
||||
for (const target of targets) {
|
||||
for (const target of sessionTargets) {
|
||||
targetStatuses.push({
|
||||
target,
|
||||
healthy: await targetHealthy(target),
|
||||
@@ -1608,7 +1614,7 @@ export function createTkmindProxy({
|
||||
}
|
||||
|
||||
async function resolveTarget(sessionId) {
|
||||
if (targets.length <= 1 || !sessionId) return primaryTarget;
|
||||
if (sessionTargets.length <= 1 || !sessionId) return primaryTarget;
|
||||
try {
|
||||
const routedTarget = await runtimeRouter?.resolveSessionTarget(sessionId);
|
||||
if (routedTarget) return routedTarget;
|
||||
@@ -1616,9 +1622,9 @@ export function createTkmindProxy({
|
||||
// Prefer the pinned URL: stable across reordering/resizing the target list.
|
||||
// Only honor it if that upstream is still configured; otherwise fall back to
|
||||
// the legacy integer index, then to primary.
|
||||
if (target && targets.includes(target)) return target;
|
||||
if (target && sessionTargets.includes(target)) return target;
|
||||
if (brokerMetrics) {
|
||||
if (target && !targets.includes(target)) {
|
||||
if (target) {
|
||||
brokerMetrics.resolveTargetMiss({
|
||||
sessionId,
|
||||
target,
|
||||
|
||||
Reference in New Issue
Block a user