feat(admin): add SEO/GEO publication catalog with crawler stats

Expose a DB-backed catalog of all online publications with SEO, GEO, total,
and bot view counts from h5_publication_views for the memind_adm dashboard.

Co-authored-by: Cursor <cursoragent@cursor.com>

mindspace-analytics-discovery.mjs

@@ -62,6 +62,22 @@ function matchHostRule(hostname, rules = []) {
   return '';
 }
 
+export function classifyReferrerHost(hostname) {
+  const host = normalizeHostname(hostname);
+  if (!host) {
+    return { discovery_channel: 'direct', discovery_source: 'direct' };
+  }
+  const geoSource = matchHostRule(host, GEO_HOST_RULES);
+  if (geoSource) {
+    return { discovery_channel: 'geo', discovery_source: geoSource };
+  }
+  const seoSource = matchHostRule(host, SEO_HOST_RULES);
+  if (seoSource) {
+    return { discovery_channel: 'seo', discovery_source: seoSource };
+  }
+  return { discovery_channel: 'referral', discovery_source: 'other' };
+}
+
 function normalizeDiscoveryChannel(value) {
   const channel = String(value ?? '').trim().toLowerCase();
   return channel === 'seo' || channel === 'geo' ? channel : '';

mindspace-seo-geo-admin-catalog.mjs

@@ -0,0 +1,243 @@
+import { classifyReferrerHost } from './mindspace-analytics-discovery.mjs';
+import { isPublicationIndexable, normalizePublicationSnapshot } from './mindspace-index-policy.mjs';
+
+const DEFAULT_PUBLIC_HOST = 'm.tkmind.cn';
+const SORT_COLUMNS = new Set([
+  'publishedAt',
+  'pageTitle',
+  'ownerLabel',
+  'totalViews',
+  'seoViews',
+  'geoViews',
+  'crawlerViews',
+  'lifetimeViewCount',
+]);
+
+function asPositiveInt(value, fallback) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed < 1) return fallback;
+  return Math.floor(parsed);
+}
+
+function normalizeSortBy(value, fallback = 'publishedAt') {
+  const sortBy = String(value ?? '').trim();
+  return SORT_COLUMNS.has(sortBy) ? sortBy : fallback;
+}
+
+function normalizeSortOrder(value, fallback = 'desc') {
+  return String(value ?? '').trim().toLowerCase() === 'asc' ? 'asc' : fallback;
+}
+
+export function resolveAdminCatalogPublicUrl(publicUrl, { publicHost = DEFAULT_PUBLIC_HOST } = {}) {
+  const raw = String(publicUrl ?? '').trim();
+  if (!raw) return '';
+  if (/^https?:\/\//i.test(raw)) return raw.split('#')[0];
+  if (raw.startsWith('/')) return `https://${publicHost}${raw}`.split('#')[0];
+  return raw.split('#')[0];
+}
+
+function compareRows(a, b, sortBy, sortOrder) {
+  const direction = sortOrder === 'asc' ? 1 : -1;
+  const left = a[sortBy];
+  const right = b[sortBy];
+  if (typeof left === 'string' || typeof right === 'string') {
+    return String(left ?? '').localeCompare(String(right ?? ''), 'zh-CN') * direction;
+  }
+  return ((Number(left) || 0) - (Number(right) || 0)) * direction;
+}
+
+function aggregateViewStats(rows = []) {
+  const stats = {
+    totalViews: 0,
+    seoViews: 0,
+    geoViews: 0,
+    crawlerViews: 0,
+  };
+  for (const row of rows) {
+    const views = Number(row.views) || 0;
+    if (views <= 0) continue;
+    stats.totalViews += views;
+    if (String(row.device_type ?? '') === 'bot') {
+      stats.crawlerViews += views;
+      continue;
+    }
+    const discovery = classifyReferrerHost(row.referrer_host);
+    if (discovery.discovery_channel === 'seo') stats.seoViews += views;
+    if (discovery.discovery_channel === 'geo') stats.geoViews += views;
+  }
+  return stats;
+}
+
+export async function listAdminSeoGeoPublicationCatalog(
+  pool,
+  {
+    page = 1,
+    pageSize = 20,
+    sortBy = 'publishedAt',
+    sortOrder = 'desc',
+    startAt = null,
+    endAt = null,
+    search = '',
+    publicHost = DEFAULT_PUBLIC_HOST,
+    now = Date.now(),
+  } = {},
+) {
+  if (!pool) {
+    return { data: [], total: 0, page: 1, pageSize: 20, totals: emptyTotals() };
+  }
+
+  const safePage = asPositiveInt(page, 1);
+  const safePageSize = Math.min(asPositiveInt(pageSize, 20), 100);
+  const normalizedSortBy = normalizeSortBy(sortBy);
+  const normalizedSortOrder = normalizeSortOrder(sortOrder);
+  const searchTerm = String(search ?? '').trim();
+  const rangeStart = Number(startAt);
+  const rangeEnd = Number(endAt);
+  const hasRange = Number.isFinite(rangeStart) && Number.isFinite(rangeEnd) && rangeEnd >= rangeStart;
+
+  const params = [];
+  let whereSql = `WHERE pr.status = 'online'`;
+  if (searchTerm) {
+    whereSql += ` AND (
+      p.title LIKE ?
+      OR u.username LIKE ?
+      OR u.display_name LIKE ?
+      OR pr.public_url LIKE ?
+    )`;
+    const like = `%${searchTerm}%`;
+    params.push(like, like, like, like);
+  }
+
+  const [countRows] = await pool.query(
+    `SELECT COUNT(*) AS total
+     FROM h5_publish_records pr
+     JOIN h5_page_records p ON p.id = pr.page_id
+     JOIN h5_users u ON u.id = pr.user_id
+     ${whereSql}`,
+    params,
+  );
+  const total = Number(countRows[0]?.total ?? 0);
+
+  const [publicationRows] = await pool.query(
+    `SELECT
+        pr.id AS publicationId,
+        pr.page_id AS pageId,
+        pr.user_id AS userId,
+        pr.public_url AS publicUrl,
+        pr.access_mode AS accessMode,
+        pr.status,
+        pr.user_confirmed_at AS userConfirmedAt,
+        pr.expires_at AS expiresAt,
+        pr.published_at AS publishedAt,
+        pr.view_count AS lifetimeViewCount,
+        p.title AS pageTitle,
+        p.summary AS pageSummary,
+        u.username,
+        u.display_name AS displayName
+     FROM h5_publish_records pr
+     JOIN h5_page_records p ON p.id = pr.page_id
+     JOIN h5_users u ON u.id = pr.user_id
+     ${whereSql}`,
+    params,
+  );
+
+  const publicationIds = publicationRows.map((row) => row.publicationId);
+  const statsByPublication = new Map();
+  if (publicationIds.length > 0 && hasRange) {
+    const placeholders = publicationIds.map(() => '?').join(', ');
+    const [viewRows] = await pool.query(
+      `SELECT publish_id AS publicationId,
+              device_type,
+              referrer_host,
+              COUNT(*) AS views
+       FROM h5_publication_views
+       WHERE publish_id IN (${placeholders})
+         AND viewed_at >= ?
+         AND viewed_at <= ?
+       GROUP BY publish_id, device_type, referrer_host`,
+      [...publicationIds, rangeStart, rangeEnd],
+    );
+    for (const row of viewRows) {
+      const key = String(row.publicationId);
+      const bucket = statsByPublication.get(key) ?? [];
+      bucket.push(row);
+      statsByPublication.set(key, bucket);
+    }
+  }
+
+  const rows = publicationRows.map((row) => {
+    const publication = normalizePublicationSnapshot({
+      id: row.publicationId,
+      pageId: row.pageId,
+      publicUrl: row.publicUrl,
+      accessMode: row.accessMode,
+      status: row.status,
+      userConfirmedAt: row.userConfirmedAt,
+      expiresAt: row.expiresAt,
+      title: row.pageTitle,
+      summary: row.pageSummary,
+    });
+    const stats = hasRange
+      ? aggregateViewStats(statsByPublication.get(String(row.publicationId)) ?? [])
+      : {
+          totalViews: 0,
+          seoViews: 0,
+          geoViews: 0,
+          crawlerViews: 0,
+        };
+    const ownerLabel =
+      String(row.displayName ?? '').trim() ||
+      String(row.username ?? '').trim() ||
+      '未命名用户';
+    return {
+      publicationId: row.publicationId,
+      pageId: row.pageId,
+      userId: row.userId,
+      ownerLabel,
+      username: String(row.username ?? '').trim(),
+      pageTitle: String(row.pageTitle ?? '').trim() || '未命名页面',
+      pageSummary: String(row.pageSummary ?? '').trim(),
+      publicUrl: resolveAdminCatalogPublicUrl(row.publicUrl, { publicHost }),
+      accessMode: row.accessMode,
+      indexable: isPublicationIndexable(publication, { now }),
+      publishedAt: Number(row.publishedAt) || null,
+      lifetimeViewCount: Number(row.lifetimeViewCount) || 0,
+      ...stats,
+    };
+  });
+
+  rows.sort((a, b) => {
+    const primary = compareRows(a, b, normalizedSortBy, normalizedSortOrder);
+    if (primary !== 0) return primary;
+    return compareRows(a, b, 'pageTitle', 'asc');
+  });
+
+  const offset = (safePage - 1) * safePageSize;
+  const pageRows = rows.slice(offset, offset + safePageSize);
+  const totals = rows.reduce(
+    (acc, row) => ({
+      totalViews: acc.totalViews + row.totalViews,
+      seoViews: acc.seoViews + row.seoViews,
+      geoViews: acc.geoViews + row.geoViews,
+      crawlerViews: acc.crawlerViews + row.crawlerViews,
+    }),
+    emptyTotals(),
+  );
+
+  return {
+    data: pageRows,
+    total,
+    page: safePage,
+    pageSize: safePageSize,
+    totals,
+  };
+}
+
+function emptyTotals() {
+  return {
+    totalViews: 0,
+    seoViews: 0,
+    geoViews: 0,
+    crawlerViews: 0,
+  };
+}

mindspace-seo-geo-admin-catalog.test.mjs

@@ -0,0 +1,113 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { classifyReferrerHost } from './mindspace-analytics-discovery.mjs';
+import {
+  listAdminSeoGeoPublicationCatalog,
+  resolveAdminCatalogPublicUrl,
+} from './mindspace-seo-geo-admin-catalog.mjs';
+
+test('classifyReferrerHost maps search and ai referrers', () => {
+  assert.deepEqual(classifyReferrerHost('www.google.com'), {
+    discovery_channel: 'seo',
+    discovery_source: 'google',
+  });
+  assert.deepEqual(classifyReferrerHost('chatgpt.com'), {
+    discovery_channel: 'geo',
+    discovery_source: 'chatgpt',
+  });
+});
+
+test('resolveAdminCatalogPublicUrl normalizes relative public urls', () => {
+  assert.equal(
+    resolveAdminCatalogPublicUrl('/u/john/pages/demo'),
+    'https://m.tkmind.cn/u/john/pages/demo',
+  );
+});
+
+test('listAdminSeoGeoPublicationCatalog merges zero-traffic pages with view stats', async () => {
+  const calls = [];
+  const pool = {
+    async query(sql, params = []) {
+      calls.push({ sql, params });
+      const normalized = String(sql).replace(/\s+/g, ' ').trim();
+      if (normalized.startsWith('SELECT COUNT(*) AS total')) {
+        return [[{ total: 2 }]];
+      }
+      if (normalized.includes('FROM h5_publish_records pr') && normalized.includes('JOIN h5_users')) {
+        return [[
+          {
+            publicationId: 'pub-1',
+            pageId: 'page-1',
+            userId: 'user-1',
+            publicUrl: '/u/john/pages/demo',
+            accessMode: 'public',
+            status: 'online',
+            userConfirmedAt: 1000,
+            expiresAt: null,
+            publishedAt: 2000,
+            lifetimeViewCount: 12,
+            pageTitle: 'Demo',
+            pageSummary: 'Summary',
+            username: 'john',
+            displayName: 'John',
+          },
+          {
+            publicationId: 'pub-2',
+            pageId: 'page-2',
+            userId: 'user-2',
+            publicUrl: '/u/jane/pages/empty',
+            accessMode: 'public',
+            status: 'online',
+            userConfirmedAt: null,
+            expiresAt: null,
+            publishedAt: 1500,
+            lifetimeViewCount: 0,
+            pageTitle: 'Empty',
+            pageSummary: '',
+            username: 'jane',
+            displayName: 'Jane',
+          },
+        ]];
+      }
+      if (normalized.includes('FROM h5_publication_views')) {
+        return [[
+          {
+            publicationId: 'pub-1',
+            device_type: 'desktop',
+            referrer_host: 'www.google.com',
+            views: 3,
+          },
+          {
+            publicationId: 'pub-1',
+            device_type: 'bot',
+            referrer_host: null,
+            views: 2,
+          },
+        ]];
+      }
+      return [[]];
+    },
+  };
+
+  const result = await listAdminSeoGeoPublicationCatalog(pool, {
+    page: 1,
+    pageSize: 10,
+    startAt: 1,
+    endAt: 9_999_999_999_999,
+    sortBy: 'totalViews',
+    sortOrder: 'desc',
+  });
+
+  assert.equal(result.total, 2);
+  assert.equal(result.data.length, 2);
+  assert.equal(result.data[0].publicationId, 'pub-1');
+  assert.equal(result.data[0].seoViews, 3);
+  assert.equal(result.data[0].crawlerViews, 2);
+  assert.equal(result.data[0].totalViews, 5);
+  assert.equal(result.data[0].indexable, true);
+  assert.equal(result.data[1].publicationId, 'pub-2');
+  assert.equal(result.data[1].totalViews, 0);
+  assert.equal(result.data[1].indexable, false);
+  assert.match(result.data[1].publicUrl, /^https:\/\/m\.tkmind\.cn\//);
+});