fix: harden public download companion sync
This commit is contained in:
@@ -2,6 +2,7 @@ import fs from 'node:fs';
|
|||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
|
|
||||||
import { extractStaticPageLinks } from './mindspace-chat-save.mjs';
|
import { extractStaticPageLinks } from './mindspace-chat-save.mjs';
|
||||||
|
import { DOWNLOADABLE_FILE_PATTERN } from './mindspace-html-download-links.mjs';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Public HTML finish-sync invariants (regression guard — do not simplify away).
|
* Public HTML finish-sync invariants (regression guard — do not simplify away).
|
||||||
@@ -10,45 +11,56 @@ import { extractStaticPageLinks } from './mindspace-chat-save.mjs';
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
const PUBLIC_HTML_PATH_PATTERN = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/i;
|
const PUBLIC_HTML_PATH_PATTERN = /(?:^|[^a-z0-9_./-])(public\/[a-z0-9][a-z0-9._/-]{0,255}\.html)\b/i;
|
||||||
const PUBLIC_DOCX_HREF_PATTERN = /href=["']([^"'#?\s]+\.docx)["']/gi;
|
const PUBLIC_DOWNLOAD_ATTR_PATTERN = /(?:href|src)=["']([^"'#?\s]+)["']/gi;
|
||||||
|
const DEFAULT_DOWNLOAD_SYNC_MIN_SIZE = 1024;
|
||||||
|
|
||||||
function normalizePublicDocxHref(href) {
|
function normalizePublicDownloadHref(href) {
|
||||||
const clean = String(href ?? '')
|
const clean = String(href ?? '')
|
||||||
.split('?')[0]
|
.split('?')[0]
|
||||||
.split('#')[0]
|
.split('#')[0]
|
||||||
.replace(/^\.\//, '')
|
.replace(/^\.\//, '')
|
||||||
.trim();
|
.trim();
|
||||||
if (!clean || clean.includes('://') || clean.startsWith('data:')) return null;
|
if (
|
||||||
|
!clean ||
|
||||||
|
clean.includes('://') ||
|
||||||
|
clean.startsWith('data:') ||
|
||||||
|
!DOWNLOADABLE_FILE_PATTERN.test(clean)
|
||||||
|
) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (clean.startsWith('../oa/')) return clean.slice(3);
|
if (clean.startsWith('../oa/')) return clean.slice(3);
|
||||||
if (clean.startsWith('oa/')) return clean;
|
if (clean.startsWith('oa/')) return clean;
|
||||||
if (!clean.includes('/')) return `public/${clean}`;
|
if (!clean.includes('/')) return `public/${clean}`;
|
||||||
return clean;
|
return clean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function extractPublicDocxReferencesFromHtml(publishDir) {
|
function extractPublicDownloadReferencesFromHtml(publishDir) {
|
||||||
const root = path.resolve(String(publishDir ?? ''));
|
const root = path.resolve(String(publishDir ?? ''));
|
||||||
const publicDir = path.join(root, 'public');
|
const publicDir = path.join(root, 'public');
|
||||||
if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) return [];
|
if (!fs.existsSync(publicDir) || !fs.statSync(publicDir).isDirectory()) return [];
|
||||||
|
|
||||||
const refs = new Set();
|
const refs = new Map();
|
||||||
for (const file of fs.readdirSync(publicDir)) {
|
for (const file of fs.readdirSync(publicDir)) {
|
||||||
if (!file.toLowerCase().endsWith('.html')) continue;
|
if (!file.toLowerCase().endsWith('.html')) continue;
|
||||||
const content = fs.readFileSync(path.join(publicDir, file), 'utf8');
|
const content = fs.readFileSync(path.join(publicDir, file), 'utf8');
|
||||||
for (const match of content.matchAll(PUBLIC_DOCX_HREF_PATTERN)) {
|
const htmlRelativePath = `public/${file}`;
|
||||||
const relativePath = normalizePublicDocxHref(match[1]);
|
for (const match of content.matchAll(PUBLIC_DOWNLOAD_ATTR_PATTERN)) {
|
||||||
if (relativePath) refs.add(relativePath);
|
const relativePath = normalizePublicDownloadHref(match[1]);
|
||||||
|
if (relativePath && !refs.has(relativePath)) {
|
||||||
|
refs.set(relativePath, { relativePath, htmlRelativePath });
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return [...refs];
|
return [...refs.values()];
|
||||||
}
|
}
|
||||||
|
|
||||||
function findLatestCompleteOaDocx(publishDir, { minSize = 8000 } = {}) {
|
function listOaDownloadCandidates(publishDir, extension, { minSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE } = {}) {
|
||||||
const oaDir = path.join(path.resolve(String(publishDir ?? '')), 'oa');
|
const oaDir = path.join(path.resolve(String(publishDir ?? '')), 'oa');
|
||||||
if (!fs.existsSync(oaDir) || !fs.statSync(oaDir).isDirectory()) return null;
|
if (!fs.existsSync(oaDir) || !fs.statSync(oaDir).isDirectory()) return [];
|
||||||
|
|
||||||
const candidates = fs
|
return fs
|
||||||
.readdirSync(oaDir)
|
.readdirSync(oaDir)
|
||||||
.filter((name) => name.toLowerCase().endsWith('.docx'))
|
.filter((name) => name.toLowerCase().endsWith(extension))
|
||||||
.map((name) => {
|
.map((name) => {
|
||||||
const absolutePath = path.join(oaDir, name);
|
const absolutePath = path.join(oaDir, name);
|
||||||
const stat = fs.statSync(absolutePath);
|
const stat = fs.statSync(absolutePath);
|
||||||
@@ -56,46 +68,105 @@ function findLatestCompleteOaDocx(publishDir, { minSize = 8000 } = {}) {
|
|||||||
})
|
})
|
||||||
.filter((item) => item.size >= minSize)
|
.filter((item) => item.size >= minSize)
|
||||||
.sort((a, b) => b.mtimeMs - a.mtimeMs || b.size - a.size);
|
.sort((a, b) => b.mtimeMs - a.mtimeMs || b.size - a.size);
|
||||||
|
|
||||||
return candidates[0] ?? null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function syncPublicDocxDownloads({ publishDir, minCompleteSize = 8000 } = {}) {
|
function resolvePublicDownloadSyncSource(
|
||||||
const root = path.resolve(String(publishDir ?? ''));
|
root,
|
||||||
if (!root) return { synced: [], skipped: [] };
|
relativePath,
|
||||||
|
htmlRelativePath,
|
||||||
|
{ minSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE } = {},
|
||||||
|
) {
|
||||||
|
const extension = path.extname(relativePath).toLowerCase();
|
||||||
|
const basename = path.basename(relativePath);
|
||||||
|
const htmlStemFilename = path.posix.basename(htmlRelativePath).replace(/\.html$/i, extension);
|
||||||
|
const exactCandidates = [
|
||||||
|
path.posix.join('oa', basename),
|
||||||
|
path.posix.join('oa', htmlStemFilename),
|
||||||
|
];
|
||||||
|
|
||||||
const source = findLatestCompleteOaDocx(root, { minSize: minCompleteSize });
|
for (const candidate of exactCandidates) {
|
||||||
if (!source) return { synced: [], skipped: [] };
|
const absolutePath = path.resolve(root, candidate);
|
||||||
|
if (absolutePath === path.resolve(root, relativePath)) continue;
|
||||||
|
try {
|
||||||
|
const stat = fs.statSync(absolutePath);
|
||||||
|
if (stat.isFile() && stat.size >= minSize) {
|
||||||
|
return {
|
||||||
|
name: path.basename(candidate),
|
||||||
|
absolutePath,
|
||||||
|
size: stat.size,
|
||||||
|
strategy: 'exact_oa_match',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore missing exact candidates and continue.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fallbackCandidates = listOaDownloadCandidates(root, extension, { minSize });
|
||||||
|
if (fallbackCandidates.length === 1) {
|
||||||
|
return { ...fallbackCandidates[0], strategy: 'single_oa_fallback' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function syncPublicDocxDownloads({
|
||||||
|
publishDir,
|
||||||
|
minCompleteSize = DEFAULT_DOWNLOAD_SYNC_MIN_SIZE,
|
||||||
|
} = {}) {
|
||||||
|
const root = path.resolve(String(publishDir ?? ''));
|
||||||
|
if (!root) return { synced: [], skipped: [], missing: [], source: null, sourceByRelativePath: {} };
|
||||||
|
|
||||||
const synced = [];
|
const synced = [];
|
||||||
const skipped = [];
|
const skipped = [];
|
||||||
for (const relativePath of extractPublicDocxReferencesFromHtml(root)) {
|
const missing = [];
|
||||||
|
const sourceByRelativePath = {};
|
||||||
|
for (const reference of extractPublicDownloadReferencesFromHtml(root)) {
|
||||||
|
const { relativePath, htmlRelativePath } = reference;
|
||||||
const destination = path.resolve(root, relativePath);
|
const destination = path.resolve(root, relativePath);
|
||||||
if (destination !== root && !destination.startsWith(`${root}${path.sep}`)) {
|
if (destination !== root && !destination.startsWith(`${root}${path.sep}`)) {
|
||||||
skipped.push(relativePath);
|
skipped.push(relativePath);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let shouldCopy = !fs.existsSync(destination) || !fs.statSync(destination).isFile();
|
const source = resolvePublicDownloadSyncSource(root, relativePath, htmlRelativePath, {
|
||||||
if (!shouldCopy) {
|
minSize: minCompleteSize,
|
||||||
try {
|
});
|
||||||
shouldCopy = fs.statSync(destination).size < source.size;
|
const destinationExists = fs.existsSync(destination) && fs.statSync(destination).isFile();
|
||||||
} catch {
|
if (destinationExists && !source) {
|
||||||
shouldCopy = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!shouldCopy) {
|
|
||||||
skipped.push(relativePath);
|
skipped.push(relativePath);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if (!source) {
|
||||||
|
missing.push(relativePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (destinationExists) {
|
||||||
|
try {
|
||||||
|
if (fs.statSync(destination).size >= source.size) {
|
||||||
|
skipped.push(relativePath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Fall through and rewrite from source.
|
||||||
|
}
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||||
fs.copyFileSync(source.absolutePath, destination);
|
fs.copyFileSync(source.absolutePath, destination);
|
||||||
synced.push(relativePath);
|
synced.push(relativePath);
|
||||||
|
sourceByRelativePath[relativePath] = source.name;
|
||||||
} catch {
|
} catch {
|
||||||
skipped.push(relativePath);
|
skipped.push(relativePath);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return { synced, skipped, source: source.name };
|
const sourceNames = [...new Set(Object.values(sourceByRelativePath))];
|
||||||
|
return {
|
||||||
|
synced,
|
||||||
|
skipped,
|
||||||
|
missing,
|
||||||
|
source: sourceNames.length === 1 ? sourceNames[0] : null,
|
||||||
|
sourceByRelativePath,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function messageText(message) {
|
function messageText(message) {
|
||||||
|
|||||||
@@ -258,6 +258,58 @@ test('syncPublicDocxDownloads skips public docx that is already up to date', ()
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('syncPublicDocxDownloads copies same-named oa docx even when it is small', () => {
|
||||||
|
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-docx-sync-'));
|
||||||
|
try {
|
||||||
|
const oaDir = path.join(publishDir, 'oa');
|
||||||
|
const publicDir = path.join(publishDir, 'public');
|
||||||
|
fs.mkdirSync(oaDir, { recursive: true });
|
||||||
|
fs.mkdirSync(publicDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(oaDir, 'industry-distribution-report.docx'), 'x'.repeat(3294));
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(publicDir, 'industry-distribution-report.html'),
|
||||||
|
'<a href="industry-distribution-report.docx" download>download</a>',
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = syncPublicDocxDownloads({ publishDir, minCompleteSize: 1024 });
|
||||||
|
assert.deepEqual(result.synced, ['public/industry-distribution-report.docx']);
|
||||||
|
assert.deepEqual(result.missing, []);
|
||||||
|
assert.equal(result.source, 'industry-distribution-report.docx');
|
||||||
|
assert.equal(
|
||||||
|
fs.readFileSync(path.join(publicDir, 'industry-distribution-report.docx'), 'utf8').length,
|
||||||
|
3294,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(publishDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('syncPublicDocxDownloads reports missing public docx when no safe source exists', () => {
|
||||||
|
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-docx-sync-'));
|
||||||
|
try {
|
||||||
|
const oaDir = path.join(publishDir, 'oa');
|
||||||
|
const publicDir = path.join(publishDir, 'public');
|
||||||
|
fs.mkdirSync(oaDir, { recursive: true });
|
||||||
|
fs.mkdirSync(publicDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(oaDir, '暑假计划.docx'), 'x'.repeat(4084));
|
||||||
|
fs.writeFileSync(path.join(oaDir, '韩国旅游攻略.docx'), 'x'.repeat(5889));
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(publicDir, 'industry-distribution-report.html'),
|
||||||
|
'<a href="industry-distribution-report.docx" download>download</a>',
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = syncPublicDocxDownloads({ publishDir, minCompleteSize: 1024 });
|
||||||
|
assert.deepEqual(result.synced, []);
|
||||||
|
assert.deepEqual(result.missing, ['public/industry-distribution-report.docx']);
|
||||||
|
assert.equal(
|
||||||
|
fs.existsSync(path.join(publicDir, 'industry-distribution-report.docx')),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(publishDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test('materializeMissingPublicHtmlWrites replaces existing real html when content changed', () => {
|
test('materializeMissingPublicHtmlWrites replaces existing real html when content changed', () => {
|
||||||
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-finish-sync-'));
|
const publishDir = fs.mkdtempSync(path.join(os.tmpdir(), 'public-finish-sync-'));
|
||||||
try {
|
try {
|
||||||
|
|||||||
+6
-1
@@ -4276,7 +4276,7 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
|
|||||||
messages = null;
|
messages = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await syncPublicHtmlAfterFinish({
|
const syncResult = await syncPublicHtmlAfterFinish({
|
||||||
messages,
|
messages,
|
||||||
currentUser: req.currentUser,
|
currentUser: req.currentUser,
|
||||||
publishDir,
|
publishDir,
|
||||||
@@ -4285,6 +4285,11 @@ api.get('/sessions/:sessionId/events', async (req, res, next) => {
|
|||||||
? (userId, options) => mindSpaceAssets.syncWorkspaceAssets(userId, options)
|
? (userId, options) => mindSpaceAssets.syncWorkspaceAssets(userId, options)
|
||||||
: null,
|
: null,
|
||||||
});
|
});
|
||||||
|
if (Array.isArray(syncResult?.docxSync?.missing) && syncResult.docxSync.missing.length > 0) {
|
||||||
|
console.warn(
|
||||||
|
`[MindSpace] missing public download files after finish for user ${uid}: ${syncResult.docxSync.missing.join(', ')}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
await syncUserGeneratedPages(uid);
|
await syncUserGeneratedPages(uid);
|
||||||
};
|
};
|
||||||
return tkmindProxy.proxySessionEvents(req, res, sessionId, {
|
return tkmindProxy.proxySessionEvents(req, res, sessionId, {
|
||||||
|
|||||||
Reference in New Issue
Block a user