Files
memind/mindspace-analytics-discovery.mjs
T
john fdc234e5d0
Memind CI / Test, build, and release guards (push) Successful in 3m38s
feat(admin): add SEO/GEO publication catalog with crawler stats
Expose a DB-backed catalog of all online publications with SEO, GEO, total,
and bot view counts from h5_publication_views for the memind_adm dashboard.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-12 16:41:39 +08:00

194 lines
8.1 KiB
JavaScript

export const DISCOVERY_QUERY_PARAM = 'memind_discovery';
export const DISCOVERY_SOURCE_QUERY_PARAM = 'memind_discovery_source';
const SEO_HOST_RULES = [
{ source: 'google', hosts: ['google.com', 'google.com.hk', 'google.co.jp', 'google.co.uk', 'google.ca', 'google.com.tw'] },
{ source: 'baidu', hosts: ['baidu.com'] },
{ source: 'bing', hosts: ['bing.com'] },
{ source: 'sogou', hosts: ['sogou.com'] },
{ source: 'so360', hosts: ['so.com'] },
{ source: 'sm', hosts: ['sm.cn'] },
{ source: 'yandex', hosts: ['yandex.ru', 'yandex.com', 'yandex.by'] },
{ source: 'duckduckgo', hosts: ['duckduckgo.com'] },
{ source: 'yahoo', hosts: ['search.yahoo.com', 'yahoo.com'] },
{ source: 'naver', hosts: ['search.naver.com', 'naver.com'] },
{ source: 'ecosia', hosts: ['ecosia.org'] },
{ source: 'brave', hosts: ['search.brave.com'] },
];
const GEO_HOST_RULES = [
{ source: 'chatgpt', hosts: ['chatgpt.com', 'chat.openai.com'] },
{ source: 'openai', hosts: ['openai.com'] },
{ source: 'perplexity', hosts: ['perplexity.ai'] },
{ source: 'claude', hosts: ['claude.ai', 'anthropic.com'] },
{ source: 'gemini', hosts: ['gemini.google.com'] },
{ source: 'copilot', hosts: ['copilot.microsoft.com'] },
{ source: 'you', hosts: ['you.com'] },
{ source: 'phind', hosts: ['phind.com'] },
{ source: 'kagi', hosts: ['kagi.com'] },
{ source: 'poe', hosts: ['poe.com'] },
{ source: 'meta_ai', hosts: ['meta.ai'] },
{ source: 'doubao', hosts: ['doubao.com'] },
{ source: 'tongyi', hosts: ['tongyi.aliyun.com', 'qianwen.aliyun.com'] },
{ source: 'deepseek', hosts: ['chat.deepseek.com', 'deepseek.com'] },
];
function normalizeHostname(value) {
return String(value ?? '').trim().toLowerCase().replace(/^www\./, '').slice(0, 255);
}
function parseReferrerHost(referrer) {
const raw = String(referrer ?? '').trim();
if (!raw) return '';
try {
return normalizeHostname(new URL(raw).hostname);
} catch {
return '';
}
}
function matchHostRule(hostname, rules = []) {
const host = normalizeHostname(hostname);
if (!host) return '';
for (const rule of rules) {
for (const candidate of rule.hosts) {
const normalized = normalizeHostname(candidate);
if (!normalized) continue;
if (host === normalized || host.endsWith(`.${normalized}`)) {
return rule.source;
}
}
}
return '';
}
export function classifyReferrerHost(hostname) {
const host = normalizeHostname(hostname);
if (!host) {
return { discovery_channel: 'direct', discovery_source: 'direct' };
}
const geoSource = matchHostRule(host, GEO_HOST_RULES);
if (geoSource) {
return { discovery_channel: 'geo', discovery_source: geoSource };
}
const seoSource = matchHostRule(host, SEO_HOST_RULES);
if (seoSource) {
return { discovery_channel: 'seo', discovery_source: seoSource };
}
return { discovery_channel: 'referral', discovery_source: 'other' };
}
function normalizeDiscoveryChannel(value) {
const channel = String(value ?? '').trim().toLowerCase();
return channel === 'seo' || channel === 'geo' ? channel : '';
}
function classifyByUtm(utmSource = '', utmMedium = '') {
const source = String(utmSource ?? '').trim().toLowerCase();
const medium = String(utmMedium ?? '').trim().toLowerCase();
if (!source && !medium) return null;
if (medium === 'organic' || /seo|search|sem/.test(source)) {
return { discovery_channel: 'seo', discovery_source: source || 'utm' };
}
if (/geo|ai|llm|gpt|copilot|perplexity|chatgpt/.test(source)) {
return { discovery_channel: 'geo', discovery_source: source || 'utm' };
}
return null;
}
export function classifyDiscoveryTrafficSource({
referrer = '',
utmSource = '',
utmMedium = '',
discoveryChannel = '',
discoverySource = '',
pageHost = '',
} = {}) {
const normalizedChannel = normalizeDiscoveryChannel(discoveryChannel);
if (normalizedChannel) {
const source = String(discoverySource ?? '').trim().toLowerCase().slice(0, 32) || 'other';
return {
discovery_channel: normalizedChannel,
discovery_source: source,
referrer_host: parseReferrerHost(referrer),
};
}
const utmMatch = classifyByUtm(utmSource, utmMedium);
const referrerHost = parseReferrerHost(referrer);
const currentHost = normalizeHostname(pageHost);
const externalHost =
referrerHost && (!currentHost || referrerHost !== currentHost)
? referrerHost
: '';
if (utmMatch) {
return {
discovery_channel: utmMatch.discovery_channel,
discovery_source: utmMatch.discovery_source,
referrer_host: externalHost,
};
}
const geoSource = matchHostRule(externalHost, GEO_HOST_RULES);
if (geoSource) {
return {
discovery_channel: 'geo',
discovery_source: geoSource,
referrer_host: externalHost,
};
}
const seoSource = matchHostRule(externalHost, SEO_HOST_RULES);
if (seoSource) {
return {
discovery_channel: 'seo',
discovery_source: seoSource,
referrer_host: externalHost,
};
}
if (externalHost) {
return {
discovery_channel: 'referral',
discovery_source: 'other',
referrer_host: externalHost,
};
}
return {
discovery_channel: 'direct',
discovery_source: 'direct',
referrer_host: '',
};
}
export function appendDiscoveryPropagationParams(url, discovery = {}) {
const channel = normalizeDiscoveryChannel(discovery.discovery_channel);
if (!channel || !url) return String(url ?? '');
let next = String(url);
const joiner = next.includes('?') ? '&' : '?';
next += `${joiner}${encodeURIComponent(DISCOVERY_QUERY_PARAM)}=${encodeURIComponent(channel)}`;
const source = String(discovery.discovery_source ?? '').trim().toLowerCase();
if (source && source !== 'direct' && source !== 'other') {
next += `&${encodeURIComponent(DISCOVERY_SOURCE_QUERY_PARAM)}=${encodeURIComponent(source.slice(0, 32))}`;
}
return next;
}
export function buildInlineDiscoveryResolverScript(referrerHint = '') {
const rules = {
seo: SEO_HOST_RULES,
geo: GEO_HOST_RULES,
};
const serializedRules = JSON.stringify(rules)
.replaceAll('<', '\\u003c')
.replaceAll('>', '\\u003e')
.replaceAll('&', '\\u0026');
const serializedHint = JSON.stringify(String(referrerHint ?? '').slice(0, 512))
.replaceAll('<', '\\u003c')
.replaceAll('>', '\\u003e')
.replaceAll('&', '\\u0026');
return `(function(){var R=${serializedRules},H=${serializedHint};function norm(v){return String(v||"").trim().toLowerCase().replace(/^www\\./,"").slice(0,255)}function hostFrom(u){try{return norm(new URL(String(u||"")).hostname)}catch(e){return""}}function matchHost(h,rules){h=norm(h);if(!h)return"";for(var i=0;i<rules.length;i++){var rule=rules[i];for(var j=0;j<rule.hosts.length;j++){var c=norm(rule.hosts[j]);if(h===c||h.endsWith("."+c))return rule.source}}return""}function byUtm(s,m){s=String(s||"").toLowerCase();m=String(m||"").toLowerCase();if(!s&&!m)return null;if(m==="organic"||/seo|search|sem/.test(s))return{discovery_channel:"seo",discovery_source:s||"utm",referrer_host:""};if(/geo|ai|llm|gpt|copilot|perplexity|chatgpt/.test(s))return{discovery_channel:"geo",discovery_source:s||"utm",referrer_host:""};return null}return function(){var p=new URLSearchParams(location.search),c=String(p.get("${DISCOVERY_QUERY_PARAM}")||"").toLowerCase(),s=String(p.get("${DISCOVERY_SOURCE_QUERY_PARAM}")||"").toLowerCase().slice(0,32);if(c==="seo"||c==="geo")return{discovery_channel:c,discovery_source:s||"other",referrer_host:""};var utm=byUtm(p.get("utm_source"),p.get("utm_medium"));if(utm){var refHost=hostFrom(document.referrer)||hostFrom(H);utm.referrer_host=refHost&&refHost!==norm(location.hostname)?refHost:"";return utm}var refHost=hostFrom(document.referrer);if(!refHost||refHost===norm(location.hostname))refHost=hostFrom(H);var geo=matchHost(refHost,R.geo);if(geo)return{discovery_channel:"geo",discovery_source:geo,referrer_host:refHost};var seo=matchHost(refHost,R.seo);if(seo)return{discovery_channel:"seo",discovery_source:seo,referrer_host:refHost};if(refHost)return{discovery_channel:"referral",discovery_source:"other",referrer_host:refHost};return{discovery_channel:"direct",discovery_source:"direct",referrer_host:""}}})()`;
}