memind/mindspace-content-scan.mjs

import crypto from 'node:crypto';

const RISK_ORDER = ['none', 'low', 'medium', 'high', 'critical'];
const HTML_TAG_PATTERN = /<[^>]+>/;
const PRIVATE_URL_PATTERN =
  /(file:\/\/[^\s"'<>]+|\/api\/mindspace\/v1\/assets\/[a-z0-9-]+(?:\/[a-z-]+)?|\/users\/[^\s"'<>]+)/gi;

const TEXT_RULES = [
  {
    type: 'id_card',
    label: '身份证号',
    riskLevel: 'high',
    blocking: true,
    pattern: /(?<!\d)\d{17}[\dXx](?!\d)/g,
    mask: (value) => `${value.slice(0, 6)}********${value.slice(-4)}`,
    replacement: (value) => `${value.slice(0, 6)}********${value.slice(-4)}`,
  },
  {
    type: 'bank_card',
    label: '银行卡号',
    riskLevel: 'high',
    blocking: true,
    pattern: /(?<!\d)\d{16,19}(?!\d)/g,
    mask: (value) => `${value.slice(0, 4)} **** **** ${value.slice(-4)}`,
    replacement: (value) => `${value.slice(0, 4)} **** **** ${value.slice(-4)}`,
  },
  {
    type: 'phone',
    label: '手机号',
    riskLevel: 'medium',
    blocking: false,
    pattern: /(?<!\d)1[3-9]\d{9}(?!\d)/g,
    mask: (value) => `${value.slice(0, 3)}****${value.slice(-4)}`,
    replacement: (value) => `${value.slice(0, 3)}****${value.slice(-4)}`,
  },
  {
    type: 'email',
    label: '邮箱地址',
    riskLevel: 'medium',
    blocking: false,
    pattern: /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
    mask: (value) => `${value.slice(0, 2)}***@***`,
    replacement: () => '[邮箱地址]',
  },
  {
    type: 'api_key',
    label: 'API Key',
    riskLevel: 'critical',
    blocking: true,
    pattern:
      /\b(?:sk|rk|pk)_(?:test_|live_|proj_)?[A-Za-z0-9]{16,}|AKIA[0-9A-Z]{16}\b|ghp_[A-Za-z0-9]{20,}\b|AIza[0-9A-Za-z\-_]{20,}\b/g,
    mask: (value) => `${value.slice(0, 4)}********${value.slice(-4)}`,
    replacement: () => '[API_KEY]',
  },
  {
    type: 'access_token',
    label: '访问令牌',
    riskLevel: 'critical',
    blocking: true,
    pattern:
      /\b(?:access[_-]?token|refresh[_-]?token|authorization)\b\s*[:=]?\s*(?:bearer\s+)?[A-Za-z0-9\-_=+.]{16,}/gi,
    mask: (value) => `${value.slice(0, 10)}********`,
    replacement: (value) => value.replace(/[A-Za-z0-9\-_=+.]{16,}$/i, '[TOKEN]'),
  },
  {
    type: 'private_key',
    label: '私钥',
    riskLevel: 'critical',
    blocking: true,
    pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
    mask: () => '-----BEGIN PRIVATE KEY-----********',
    replacement: () => '[PRIVATE_KEY]',
  },
  {
    type: 'external_link',
    label: '外部链接',
    riskLevel: 'low',
    blocking: false,
    pattern: /https?:\/\/[^\s<>"')]+/gi,
    mask: (value) => value.slice(0, 80),
    replacement: (value) => value,
  },
];

const HTML_RULES = [
  {
    type: 'html_script',
    label: '脚本标签',
    riskLevel: 'critical',
    blocking: true,
    pattern: /<script\b[\s\S]*?<\/script>/gi,
    mask: () => '<script>…</script>',
    replacement: () => '',
  },
  {
    type: 'html_inline_handler',
    label: '内联事件',
    riskLevel: 'high',
    blocking: true,
    pattern: /\son[a-z]+\s*=\s*(['"]).*?\1/gi,
    mask: () => 'on*=…',
    replacement: () => '',
  },
  {
    type: 'html_javascript_url',
    label: 'javascript 链接',
    riskLevel: 'high',
    blocking: true,
    pattern: /(href|src)\s*=\s*(['"])\s*javascript:[\s\S]*?\2/gi,
    mask: (value) => value.slice(0, 40),
    replacement: (_value, match) => `${match[1]}=${match[2]}#${match[2]}`,
  },
  {
    type: 'html_forbidden_embed',
    label: '嵌入式外部内容',
    riskLevel: 'high',
    blocking: true,
    pattern: /<(iframe|object|embed)\b[\s\S]*?>[\s\S]*?(?:<\/\1>|$)/gi,
    mask: (value, match) => `<${match[1]}>…</${match[1]}>`,
    replacement: () => '',
  },
  {
    type: 'html_form_action',
    label: '表单提交',
    riskLevel: 'high',
    blocking: true,
    pattern: /<form\b[\s\S]*?>[\s\S]*?<\/form>/gi,
    mask: () => '<form>…</form>',
    replacement: () => '',
  },
  {
    type: 'html_meta_refresh',
    label: '页面跳转',
    riskLevel: 'medium',
    blocking: true,
    pattern: /<meta\b[^>]*http-equiv\s*=\s*(['"])refresh\1[^>]*>/gi,
    mask: () => '<meta http-equiv="refresh">',
    replacement: () => '',
  },
  {
    type: 'private_resource_reference',
    label: '私有资源引用',
    riskLevel: 'high',
    blocking: true,
    pattern: PRIVATE_URL_PATTERN,
    mask: (value) => value.slice(0, 48),
    replacement: () => '#private-resource-redacted',
  },
];

const ACKNOWLEDGEABLE_HTML_ACTIVE_TYPES = new Set([
  'html_script',
  'html_inline_handler',
]);

const TRUSTED_EXTERNAL_HOSTS = new Set([
  'fonts.googleapis.com',
  'fonts.gstatic.com',
  'images.unsplash.com',
]);

export function replacePrivateResourceReferences(content, replacement) {
  const source = String(content ?? '');
  if (!source) return source;
  const replacer =
    typeof replacement === 'function' ? replacement : () => String(replacement ?? '');
  return source.replace(PRIVATE_URL_PATTERN, (...args) => replacer(...args));
}

function inferFormat(content, format) {
  if (format === 'html') return 'html';
  return HTML_TAG_PATTERN.test(String(content ?? '')) ? 'html' : 'text';
}

function isTrustedExternalUrl(value) {
  try {
    return TRUSTED_EXTERNAL_HOSTS.has(new URL(value).hostname);
  } catch {
    return false;
  }
}

function riskMax(left, right) {
  return RISK_ORDER.indexOf(right) > RISK_ORDER.indexOf(left) ? right : left;
}

function uniqueMatches(content, pattern) {
  return [
    ...new Map(
      [...String(content).matchAll(pattern)].map((match) => [
        match[0].toLowerCase(),
        match,
      ]),
    ).values(),
  ];
}

function applyRule(content, rule, findings, redactions, options = {}) {
  let matches = uniqueMatches(content, rule.pattern);
  if (rule.type === 'external_link') {
    matches = matches.filter((match) => !isTrustedExternalUrl(match[0]));
  }
  if (!matches.length) return content;
  const blocking =
    options.allowHtmlActiveContent && ACKNOWLEDGEABLE_HTML_ACTIVE_TYPES.has(rule.type)
      ? false
      : rule.blocking;
  findings.push({
    id: crypto
      .createHash('sha256')
      .update(`${rule.type}:${matches[0][0]}`)
      .digest('hex')
      .slice(0, 24),
    type: rule.type,
    label: rule.label,
    riskLevel: rule.riskLevel,
    occurrenceCount: matches.length,
    sampleMasked: rule.mask(matches[0][0], matches[0]),
    blocking,
  });
  let next = String(content);
  if (typeof rule.replacement === 'function') {
    next = next.replace(rule.pattern, (...args) => rule.replacement(...args));
    redactions.push(rule.type);
  }
  return next;
}

function finalizeResult(findings) {
  const riskLevel = findings.reduce((highest, finding) => riskMax(highest, finding.riskLevel), 'none');
  return {
    status: findings.some((finding) => finding.blocking) ? 'blocked' : findings.length ? 'warned' : 'passed',
    riskLevel,
    findings,
    allowed: !findings.some((finding) => finding.blocking),
  };
}

export function scanContent(content, options = {}) {
  const format = inferFormat(content, options.format);
  const findings = [];
  const working = String(content ?? '');
  for (const rule of TEXT_RULES) {
    applyRule(working, rule, findings, [], options);
  }
  if (format === 'html') {
    for (const rule of HTML_RULES) {
      applyRule(working, rule, findings, [], options);
    }
  }
  return finalizeResult(findings);
}

export function redactContent(content, options = {}) {
  const format = inferFormat(content, options.format);
  const findings = [];
  const redactions = [];
  let next = String(content ?? '');
  for (const rule of TEXT_RULES) {
    next = applyRule(next, rule, findings, redactions);
  }
  if (format === 'html') {
    for (const rule of HTML_RULES) {
      next = applyRule(next, rule, findings, redactions);
    }
  }
  const result = finalizeResult(findings);
  return {
    ...result,
    content: next,
    format,
    redactionCount: redactions.length,
    changed: next !== String(content ?? ''),
  };
}