From 6cb1a2475dd7aa53c681eb6ae3dd0b839a7a5f9e Mon Sep 17 00:00:00 2001 From: john Date: Wed, 1 Jul 2026 11:10:58 +0800 Subject: [PATCH] fix: harden public download companion sync --- mindspace-public-finish-sync.mjs | 133 ++++++++++++++++++++------ mindspace-public-finish-sync.test.mjs | 52 ++++++++++ server.mjs | 7 +- 3 files changed, 160 insertions(+), 32 deletions(-) diff --git a/mindspace-public-finish-sync.mjs b/mindspace-public-finish-sync.mjs index 2f5a0b6..37958e7 100644 --- a/mindspace-public-finish-sync.mjs +++ b/mindspace-public-finish-sync.mjs @@ -2,6 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import { extractStaticPageLinks } from './mindspace-chat-save.mjs'; +import { DOWNLOADABLE_FILE_PATTERN } from './mindspace-html-download-links.mjs'; /** * Public HTML finish-sync invariants (regression guard — do not simplify away). @@ -10,45 +11,56 @@ import { extractStaticPageLinks } from './mindspace-chat-save.mjs'; */ const PUBLIC_HTML_PATH_PATTERN = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/i; -const PUBLIC_DOCX_HREF_PATTERN = /href=["']([^"'#?\s]+\.docx)["']/gi; +const PUBLIC_DOWNLOAD_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi; +const DEFAULT_DOWNLOAD_SYNC_MIN_SIZE = 1024; -function normalizePublicDocxHref(href) { +function normalizePublicDownloadHref(href) { const clean = String(href ?? '') .split('?')[0] .split('#')[0] .replace(/^\.\//, '') .trim(); - if (!clean || clean.includes('://') || clean.startsWith('data:')) return null; + if ( + !clean || + clean.includes('://') || + clean.startsWith('data:') || + !DOWNLOADABLE_FILE_PATTERN.test(clean) + ) { + return null; + } if (clean.startsWith('../oa/')) return clean.slice(3); if (clean.startsWith('oa/')) return clean; if (!clean.includes('/')) return `public/${clean}`; return clean; } -function extractPublicDocxReferencesFromHtml(publishDir) { +function extractPublicDownloadReferencesFromHtml(publishDir) { const root = path.resolve(String(publishDir ?? '')); const publicDir = path.join(root, 'public'); if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) return []; - const refs = new Set(); + const refs = new Map(); for (const file of fs.readdirSync(publicDir)) { if (!file.toLowerCase().endsWith('.html')) continue; const content = fs.readFileSync(path.join(publicDir, file), 'utf8'); - for (const match of content.matchAll(PUBLIC_DOCX_HREF_PATTERN)) { - const relativePath = normalizePublicDocxHref(match[1]); - if (relativePath) refs.add(relativePath); + const htmlRelativePath = `public/${file}`; + for (const match of content.matchAll(PUBLIC_DOWNLOAD_ATTR_PATTERN)) { + const relativePath = normalizePublicDownloadHref(match[1]); + if (relativePath && !refs.has(relativePath)) { + refs.set(relativePath, { relativePath, htmlRelativePath }); + } } } - return [...refs]; + return [...refs.values()]; } -function findLatestCompleteOaDocx(publishDir, { minSize = 8000 } = {}) { +function listOaDownloadCandidates(publishDir, extension, { minSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE } = {}) { const oaDir = path.join(path.resolve(String(publishDir ?? '')), 'oa'); - if (!fs.existsSync(oaDir) || !fs.statSync(oaDir).isDirectory()) return null; + if (!fs.existsSync(oaDir) || !fs.statSync(oaDir).isDirectory()) return []; - const candidates = fs + return fs .readdirSync(oaDir) - .filter((name) => name.toLowerCase().endsWith('.docx')) + .filter((name) => name.toLowerCase().endsWith(extension)) .map((name) => { const absolutePath = path.join(oaDir, name); const stat = fs.statSync(absolutePath); @@ -56,46 +68,105 @@ function findLatestCompleteOaDocx(publishDir, { minSize = 8000 } = {}) { }) .filter((item) => item.size >= minSize) .sort((a, b) => b.mtimeMs - a.mtimeMs || b.size - a.size); - - return candidates[0] ?? null; } -export function syncPublicDocxDownloads({ publishDir, minCompleteSize = 8000 } = {}) { - const root = path.resolve(String(publishDir ?? '')); - if (!root) return { synced: [], skipped: [] }; +function resolvePublicDownloadSyncSource( + root, + relativePath, + htmlRelativePath, + { minSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE } = {}, +) { + const extension = path.extname(relativePath).toLowerCase(); + const basename = path.basename(relativePath); + const htmlStemFilename = path.posix.basename(htmlRelativePath).replace(/\.html$/i, extension); + const exactCandidates = [ + path.posix.join('oa', basename), + path.posix.join('oa', htmlStemFilename), + ]; - const source = findLatestCompleteOaDocx(root, { minSize: minCompleteSize }); - if (!source) return { synced: [], skipped: [] }; + for (const candidate of exactCandidates) { + const absolutePath = path.resolve(root, candidate); + if (absolutePath === path.resolve(root, relativePath)) continue; + try { + const stat = fs.statSync(absolutePath); + if (stat.isFile() && stat.size >= minSize) { + return { + name: path.basename(candidate), + absolutePath, + size: stat.size, + strategy: 'exact_oa_match', + }; + } + } catch { + // Ignore missing exact candidates and continue. + } + } + + const fallbackCandidates = listOaDownloadCandidates(root, extension, { minSize }); + if (fallbackCandidates.length === 1) { + return { ...fallbackCandidates[0], strategy: 'single_oa_fallback' }; + } + + return null; +} + +export function syncPublicDocxDownloads({ + publishDir, + minCompleteSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE, +} = {}) { + const root = path.resolve(String(publishDir ?? '')); + if (!root) return { synced: [], skipped: [], missing: [], source: null, sourceByRelativePath: {} }; const synced = []; const skipped = []; - for (const relativePath of extractPublicDocxReferencesFromHtml(root)) { + const missing = []; + const sourceByRelativePath = {}; + for (const reference of extractPublicDownloadReferencesFromHtml(root)) { + const { relativePath, htmlRelativePath } = reference; const destination = path.resolve(root, relativePath); if (destination !== root && !destination.startsWith(`${root}${path.sep}`)) { skipped.push(relativePath); continue; } - let shouldCopy = !fs.existsSync(destination) || !fs.statSync(destination).isFile(); - if (!shouldCopy) { - try { - shouldCopy = fs.statSync(destination).size < source.size; - } catch { - shouldCopy = true; - } - } - if (!shouldCopy) { + const source = resolvePublicDownloadSyncSource(root, relativePath, htmlRelativePath, { + minSize: minCompleteSize, + }); + const destinationExists = fs.existsSync(destination) && fs.statSync(destination).isFile(); + if (destinationExists && !source) { skipped.push(relativePath); continue; } + if (!source) { + missing.push(relativePath); + continue; + } + if (destinationExists) { + try { + if (fs.statSync(destination).size >= source.size) { + skipped.push(relativePath); + continue; + } + } catch { + // Fall through and rewrite from source. + } + } try { fs.mkdirSync(path.dirname(destination), { recursive: true }); fs.copyFileSync(source.absolutePath, destination); synced.push(relativePath); + sourceByRelativePath[relativePath] = source.name; } catch { skipped.push(relativePath); } } - return { synced, skipped, source: source.name }; + const sourceNames = [...new Set(Object.values(sourceByRelativePath))]; + return { + synced, + skipped, + missing, + source: sourceNames.length === 1 ? sourceNames[0] : null, + sourceByRelativePath, + }; } function messageText(message) { diff --git a/mindspace-public-finish-sync.test.mjs b/mindspace-public-finish-sync.test.mjs index 27b1342..dd258b0 100644 --- a/mindspace-public-finish-sync.test.mjs +++ b/mindspace-public-finish-sync.test.mjs @@ -258,6 +258,58 @@ test('syncPublicDocxDownloads skips public docx that is already up to date', () } }); +test('syncPublicDocxDownloads copies same-named oa docx even when it is small', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-docx-sync-')); + try { + const oaDir = path.join(publishDir, 'oa'); + const publicDir = path.join(publishDir, 'public'); + fs.mkdirSync(oaDir, { recursive: true }); + fs.mkdirSync(publicDir, { recursive: true }); + fs.writeFileSync(path.join(oaDir, 'industry-distribution-report.docx'), 'x'.repeat(3294)); + fs.writeFileSync( + path.join(publicDir, 'industry-distribution-report.html'), + 'download', + ); + + const result = syncPublicDocxDownloads({ publishDir, minCompleteSize: 1024 }); + assert.deepEqual(result.synced, ['public/industry-distribution-report.docx']); + assert.deepEqual(result.missing, []); + assert.equal(result.source, 'industry-distribution-report.docx'); + assert.equal( + fs.readFileSync(path.join(publicDir, 'industry-distribution-report.docx'), 'utf8').length, + 3294, + ); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + +test('syncPublicDocxDownloads reports missing public docx when no safe source exists', () => { + const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-docx-sync-')); + try { + const oaDir = path.join(publishDir, 'oa'); + const publicDir = path.join(publishDir, 'public'); + fs.mkdirSync(oaDir, { recursive: true }); + fs.mkdirSync(publicDir, { recursive: true }); + fs.writeFileSync(path.join(oaDir, '暑假计划.docx'), 'x'.repeat(4084)); + fs.writeFileSync(path.join(oaDir, '韩国旅游攻略.docx'), 'x'.repeat(5889)); + fs.writeFileSync( + path.join(publicDir, 'industry-distribution-report.html'), + 'download', + ); + + const result = syncPublicDocxDownloads({ publishDir, minCompleteSize: 1024 }); + assert.deepEqual(result.synced, []); + assert.deepEqual(result.missing, ['public/industry-distribution-report.docx']); + assert.equal( + fs.existsSync(path.join(publicDir, 'industry-distribution-report.docx')), + false, + ); + } finally { + fs.rmSync(publishDir, { recursive: true, force: true }); + } +}); + test('materializeMissingPublicHtmlWrites replaces existing real html when content changed', () => { const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-finish-sync-')); try { diff --git a/server.mjs b/server.mjs index ec69d6b..d58d06d 100644 --- a/server.mjs +++ b/server.mjs @@ -4276,7 +4276,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { messages = null; } } - await syncPublicHtmlAfterFinish({ + const syncResult = await syncPublicHtmlAfterFinish({ messages, currentUser: req.currentUser, publishDir, @@ -4285,6 +4285,11 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => { ? (userId, options) => mindSpaceAssets.syncWorkspaceAssets(userId, options) : null, }); + if (Array.isArray(syncResult?.docxSync?.missing) && syncResult.docxSync.missing.length > 0) { + console.warn( + `[MindSpace] missing public download files after finish for user ${uid}: ${syncResult.docxSync.missing.join(', ')}`, + ); + } await syncUserGeneratedPages(uid); }; return tkmindProxy.proxySessionEvents(req, res, sessionId, {