feat: add guarded portal canary release
Memind CI / Test, build, and release guards (push) Failing after 2m14s
Memind CI / Test, build, and release guards (push) Failing after 2m14s
This commit is contained in:
@@ -8,6 +8,7 @@ import { createMindSpaceAuditWriter } from '../mindspace-audit.mjs';
|
||||
import { createMindSpaceImageGenerationService } from '../mindspace-image-generation.mjs';
|
||||
import { createMindSpaceImageReviewService } from '../mindspace-image-review.mjs';
|
||||
import { createWordFilterService } from '../word-filter.mjs';
|
||||
import { isPassiveCanaryRuntime } from './portal-runtime-role.mjs';
|
||||
|
||||
async function createPgExperienceServiceFromModule(options) {
|
||||
const { createPgExperienceService } = await import(
|
||||
@@ -108,28 +109,32 @@ export async function bootstrapPortalAgentServices({
|
||||
const mindSpaceAudit =
|
||||
createMindSpaceAuditWriterFn(pool);
|
||||
|
||||
mindSpaceRuntimeAdapter.startBackgroundJobs({
|
||||
publicationCleanupIntervalMs: 60 * 1000,
|
||||
agentWorker: runtime.agentWorker,
|
||||
agentRunner: mindSpaceAgentRunner,
|
||||
workspaceMaintenanceEnabled,
|
||||
publishRoot: runtime.publishRoot,
|
||||
startWorkspaceThumbnailWatcher,
|
||||
startWorkspaceAssetSyncWatcher,
|
||||
syncUserWorkspaceByDirKey: async (
|
||||
dirKey,
|
||||
options,
|
||||
) => {
|
||||
const userId =
|
||||
await resolveUserIdByDirKey(dirKey);
|
||||
if (!userId) return;
|
||||
await mindSpaceAssets.syncWorkspaceAssets(
|
||||
userId,
|
||||
if (isPassiveCanaryRuntime(env)) {
|
||||
logger.log('Passive candidate runtime: MindSpace background jobs disabled');
|
||||
} else {
|
||||
mindSpaceRuntimeAdapter.startBackgroundJobs({
|
||||
publicationCleanupIntervalMs: 60 * 1000,
|
||||
agentWorker: runtime.agentWorker,
|
||||
agentRunner: mindSpaceAgentRunner,
|
||||
workspaceMaintenanceEnabled,
|
||||
publishRoot: runtime.publishRoot,
|
||||
startWorkspaceThumbnailWatcher,
|
||||
startWorkspaceAssetSyncWatcher,
|
||||
syncUserWorkspaceByDirKey: async (
|
||||
dirKey,
|
||||
options,
|
||||
);
|
||||
},
|
||||
expireStaleUploadsIntervalMs: 5 * 60 * 1000,
|
||||
});
|
||||
) => {
|
||||
const userId =
|
||||
await resolveUserIdByDirKey(dirKey);
|
||||
if (!userId) return;
|
||||
await mindSpaceAssets.syncWorkspaceAssets(
|
||||
userId,
|
||||
options,
|
||||
);
|
||||
},
|
||||
expireStaleUploadsIntervalMs: 5 * 60 * 1000,
|
||||
});
|
||||
}
|
||||
|
||||
await userAuth.ensureAdminUser();
|
||||
const userDataSpaceBackfill =
|
||||
|
||||
@@ -424,3 +424,25 @@ test('can disable relay bootstrap for backend-LLM-only isolated runs', async ()
|
||||
assert.equal(setup.calls.some(([name]) => name === 'relay-bootstrap'), false);
|
||||
assert.equal(setup.calls.some(([name]) => name === 'provider-sync'), true);
|
||||
});
|
||||
|
||||
test('passive candidate runtime does not start MindSpace background jobs', async () => {
|
||||
const setup = createSetup({
|
||||
env: {
|
||||
EXPERIENCE_PG_URL: 'postgres://experience',
|
||||
MEMIND_RUNTIME_ROLE: 'candidate',
|
||||
MEMIND_CANARY_PASSIVE_RUNTIME: '1',
|
||||
},
|
||||
});
|
||||
|
||||
const result = await bootstrapPortalAgentServices(setup.options);
|
||||
await Promise.all([result.relayBootstrapTask, result.providerSyncTask]);
|
||||
|
||||
assert.equal(setup.calls.some(([name]) => name === 'start-background'), false);
|
||||
assert.ok(
|
||||
setup.calls.some(
|
||||
([name, message]) =>
|
||||
name === 'log'
|
||||
&& message === 'Passive candidate runtime: MindSpace background jobs disabled',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -23,6 +23,7 @@ import { createPlazaOpsService } from '../plaza-ops.mjs';
|
||||
import { createPlazaPostService, formatPostRow } from '../plaza-posts.mjs';
|
||||
import { createPlazaRecommendService } from '../plaza-recommend.mjs';
|
||||
import { createPlazaRedis } from '../plaza-redis.mjs';
|
||||
import { isPassiveCanaryRuntime } from './portal-runtime-role.mjs';
|
||||
import { createPlazaSeoService } from '../plaza-seo.mjs';
|
||||
import {
|
||||
startPlazaTasks,
|
||||
@@ -287,12 +288,16 @@ export async function bootstrapPortalDomainServices({
|
||||
invalidateFeedCaches: () =>
|
||||
plazaRedis?.invalidateFeedCaches?.(),
|
||||
});
|
||||
startPlazaTasksFn({
|
||||
pool,
|
||||
plazaRedis,
|
||||
recalculateHotScores: recalculateHotScoresFn,
|
||||
writebackPublications: writebackPublicationsFn,
|
||||
});
|
||||
if (isPassiveCanaryRuntime(env)) {
|
||||
logger.log('Passive candidate runtime: Plaza background tasks disabled');
|
||||
} else {
|
||||
startPlazaTasksFn({
|
||||
pool,
|
||||
plazaRedis,
|
||||
recalculateHotScores: recalculateHotScoresFn,
|
||||
writebackPublications: writebackPublicationsFn,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
mindSearchConfigService,
|
||||
|
||||
@@ -416,3 +416,24 @@ test('domain bootstrap preserves directory and Plaza cross-service callbacks', a
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('passive candidate runtime does not start Plaza background tasks', async () => {
|
||||
const setup = createBootstrapSetup({
|
||||
env: {
|
||||
H5_DEFAULT_TIMEZONE: 'Asia/Singapore',
|
||||
MEMIND_RUNTIME_ROLE: 'candidate',
|
||||
MEMIND_CANARY_PASSIVE_RUNTIME: '1',
|
||||
},
|
||||
});
|
||||
|
||||
await bootstrapPortalDomainServices(setup.dependencies);
|
||||
|
||||
assert.equal(setup.calls.some((call) => call.kind === 'plaza-tasks'), false);
|
||||
assert.ok(
|
||||
setup.calls.some(
|
||||
(call) =>
|
||||
call.kind === 'log'
|
||||
&& call.message === 'Passive candidate runtime: Plaza background tasks disabled',
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@ import { createOrchestratorAdminConfigService } from '../services/orchestrator/a
|
||||
import { createWorkflowShadowObserver } from '../services/orchestrator/shadow-observer.mjs';
|
||||
import { createTkmindProxy } from '../tkmind-proxy.mjs';
|
||||
import { createToolGateway } from '../tool-gateway.mjs';
|
||||
import { isPassiveCanaryRuntime } from './portal-runtime-role.mjs';
|
||||
|
||||
function isEnabledFlag(value, fallback = '') {
|
||||
return ['1', 'true', 'yes', 'on'].includes(
|
||||
@@ -433,11 +434,12 @@ export function bootstrapPortalGatewayServices({
|
||||
),
|
||||
workerIdentity,
|
||||
});
|
||||
const agentRunRecoveryTimer =
|
||||
startAgentRunRecoveryLoopFn(
|
||||
agentRunGateway,
|
||||
{ env, logger: console },
|
||||
);
|
||||
const agentRunRecoveryTimer = isPassiveCanaryRuntime(env)
|
||||
? null
|
||||
: startAgentRunRecoveryLoopFn(
|
||||
agentRunGateway,
|
||||
{ env, logger: console },
|
||||
);
|
||||
|
||||
return {
|
||||
tkmindProxy,
|
||||
|
||||
@@ -288,6 +288,25 @@ test('preserves memory, page sync, and busy callbacks', async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('passive candidate runtime disables the singleton Agent recovery loop', () => {
|
||||
let recoveryStarts = 0;
|
||||
const setup = createSetup({
|
||||
env: {
|
||||
MEMIND_RUNTIME_ROLE: 'candidate',
|
||||
MEMIND_CANARY_PASSIVE_RUNTIME: '1',
|
||||
},
|
||||
startAgentRunRecoveryLoopFn() {
|
||||
recoveryStarts += 1;
|
||||
return { id: 'recovery' };
|
||||
},
|
||||
});
|
||||
|
||||
const result = bootstrapPortalGatewayServices(setup.options);
|
||||
|
||||
assert.equal(recoveryStarts, 0);
|
||||
assert.equal(result.agentRunRecoveryTimer, null);
|
||||
});
|
||||
|
||||
test('keeps memory observation optional and parses disabled dispatch', async () => {
|
||||
const setup = createSetup({
|
||||
env: {
|
||||
|
||||
@@ -11,6 +11,7 @@ import { createPageEditSessionService } from '../mindspace-page-edit-session.mjs
|
||||
import { resolveMindSpaceRuntimeConfig } from '../mindspace-runtime-config.mjs';
|
||||
import { createNotificationDispatcher } from '../notification-dispatcher.mjs';
|
||||
import { startScheduleReminderWorker } from '../schedule-reminder-worker.mjs';
|
||||
import { isPassiveCanaryRuntime } from './portal-runtime-role.mjs';
|
||||
import { loadWechatMpModule } from '../wechat-mp-loader.mjs';
|
||||
|
||||
export async function bootstrapPortalIntegrationServices({
|
||||
@@ -224,6 +225,7 @@ export async function bootstrapPortalIntegrationServices({
|
||||
|
||||
let scheduleReminderWorker = null;
|
||||
if (
|
||||
!isPassiveCanaryRuntime(env) &&
|
||||
env.H5_REMINDER_WORKER_ENABLED === '1' &&
|
||||
wechatMpService?.enabled &&
|
||||
scheduleService
|
||||
@@ -237,7 +239,7 @@ export async function bootstrapPortalIntegrationServices({
|
||||
}
|
||||
|
||||
let subscriptionExpiryTimer = null;
|
||||
if (subscriptionService) {
|
||||
if (subscriptionService && !isPassiveCanaryRuntime(env)) {
|
||||
subscriptionExpiryTimer = setIntervalFn(
|
||||
async () => {
|
||||
try {
|
||||
|
||||
@@ -507,3 +507,24 @@ test('keeps optional integrations disabled and contains timer failures', async (
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test('passive candidate runtime disables singleton reminder and subscription timers', async () => {
|
||||
const setup = createSetup({
|
||||
env: {
|
||||
H5_SCHEDULE_ENABLED: '1',
|
||||
H5_REMINDER_WORKER_ENABLED: '1',
|
||||
MEMIND_RUNTIME_ROLE: 'candidate',
|
||||
MEMIND_CANARY_PASSIVE_RUNTIME: '1',
|
||||
},
|
||||
});
|
||||
|
||||
const result = await bootstrapPortalIntegrationServices(setup.options);
|
||||
const captured = setup.getCaptured();
|
||||
|
||||
assert.equal(result.scheduleReminderWorker, null);
|
||||
assert.equal(result.subscriptionExpiryTimer, null);
|
||||
assert.equal(captured.reminderOptions, undefined);
|
||||
assert.equal(captured.timerCallback, undefined);
|
||||
assert.equal(setup.calls.some(([name]) => name === 'reminder-worker'), false);
|
||||
assert.equal(setup.calls.some(([name]) => name === 'set-interval'), false);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
const RUNTIME_ROLES = new Set(['stable', 'candidate']);
|
||||
|
||||
export function resolvePortalRuntimeRole(env = process.env) {
|
||||
const configured = String(env.MEMIND_RUNTIME_ROLE ?? '').trim().toLowerCase();
|
||||
return RUNTIME_ROLES.has(configured) ? configured : 'stable';
|
||||
}
|
||||
|
||||
export function isPassiveCanaryRuntime(env = process.env) {
|
||||
return (
|
||||
resolvePortalRuntimeRole(env) === 'candidate'
|
||||
&& String(env.MEMIND_CANARY_PASSIVE_RUNTIME ?? '1').trim() !== '0'
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import test from 'node:test';
|
||||
|
||||
import {
|
||||
isPassiveCanaryRuntime,
|
||||
resolvePortalRuntimeRole,
|
||||
} from './portal-runtime-role.mjs';
|
||||
|
||||
test('runtime role defaults to stable and rejects unknown values', () => {
|
||||
assert.equal(resolvePortalRuntimeRole({}), 'stable');
|
||||
assert.equal(resolvePortalRuntimeRole({ MEMIND_RUNTIME_ROLE: 'unknown' }), 'stable');
|
||||
});
|
||||
|
||||
test('candidate runtime is passive unless explicitly disabled', () => {
|
||||
assert.equal(
|
||||
isPassiveCanaryRuntime({ MEMIND_RUNTIME_ROLE: 'candidate' }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isPassiveCanaryRuntime({
|
||||
MEMIND_RUNTIME_ROLE: 'candidate',
|
||||
MEMIND_CANARY_PASSIVE_RUNTIME: '0',
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isPassiveCanaryRuntime({ MEMIND_RUNTIME_ROLE: 'stable' }),
|
||||
false,
|
||||
);
|
||||
});
|
||||
@@ -1,3 +1,5 @@
|
||||
import { resolvePortalRuntimeRole } from './portal-runtime-role.mjs';
|
||||
|
||||
function runtimeEnvFlag(value, fallback = false) {
|
||||
const raw = String(value ?? '').trim().toLowerCase();
|
||||
if (!raw) return fallback;
|
||||
@@ -54,6 +56,7 @@ export function attachPortalRuntimeRoutes(
|
||||
}
|
||||
|
||||
api.get('/status', async (_req, res, next) => {
|
||||
res.setHeader?.('X-Memind-Runtime-Role', resolvePortalRuntimeRole(env));
|
||||
await waitForUserAuthReady();
|
||||
const userAuth = getUserAuth();
|
||||
const tkmindProxy = getTkmindProxy();
|
||||
|
||||
@@ -14,6 +14,7 @@ function createRouterRecorder() {
|
||||
|
||||
function createResponseRecorder() {
|
||||
return {
|
||||
headers: new Map(),
|
||||
statusCode: 200,
|
||||
body: undefined,
|
||||
responseType: null,
|
||||
@@ -31,6 +32,10 @@ function createResponseRecorder() {
|
||||
this.body = body;
|
||||
return this;
|
||||
},
|
||||
setHeader(name, value) {
|
||||
this.headers.set(String(name).toLowerCase(), String(value));
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -73,6 +78,31 @@ test('GET /status preserves upstream status and body', async () => {
|
||||
assert.equal(res.statusCode, 207);
|
||||
assert.equal(res.responseType, 'send');
|
||||
assert.equal(res.body, 'upstream-status');
|
||||
assert.equal(res.headers.get('x-memind-runtime-role'), 'stable');
|
||||
});
|
||||
|
||||
test('GET /status identifies a passive candidate runtime without changing the body', async () => {
|
||||
const api = createRouterRecorder();
|
||||
attachPortalRuntimeRoutes(api, {
|
||||
env: { MEMIND_RUNTIME_ROLE: 'candidate' },
|
||||
getUserAuth: () => ({}),
|
||||
getTkmindProxy: () => ({
|
||||
async apiFetch() {
|
||||
return {
|
||||
status: 200,
|
||||
async text() {
|
||||
return 'ok';
|
||||
},
|
||||
};
|
||||
},
|
||||
}),
|
||||
});
|
||||
const res = createResponseRecorder();
|
||||
|
||||
await api.routes.get('/status')({}, res, () => {});
|
||||
|
||||
assert.equal(res.body, 'ok');
|
||||
assert.equal(res.headers.get('x-memind-runtime-role'), 'candidate');
|
||||
});
|
||||
|
||||
test('GET /status falls through when multi-user proxy services are unavailable', async () => {
|
||||
|
||||
Reference in New Issue
Block a user