feat: ask ai discord bot (#6842)

Signed-off-by: The-Best-Codes <bestcodes.official@gmail.com>
2026-02-02 16:11:28 -06:00
parent fafda07dd0
commit 849cc60fbc
23 changed files with 1262 additions and 0 deletions
@@ -0,0 +1,96 @@
+import { marked } from "marked";
+
+const MAX_DISCORD_LENGTH = 2000;
+
+/**
+ * Chunks markdown text intelligently, respecting markdown structure.
+ * Avoids splitting code blocks, lists, and other block elements when possible.
+ * Falls back to character-based splitting for oversized blocks.
+ *
+ * @param markdown - The markdown text to chunk
+ * @param maxLength - Maximum length per chunk (default: 2000 for Discord)
+ * @returns Array of markdown chunks
+ */
+export function chunkMarkdown(
+  markdown: string,
+  maxLength: number = MAX_DISCORD_LENGTH,
+): string[] {
+  // If text is short enough, return as-is
+  if (markdown.length <= maxLength) {
+    return [markdown];
+  }
+
+  const tokens = marked.lexer(markdown);
+  const chunks: string[] = [];
+  let currentChunk = "";
+
+  for (const token of tokens) {
+    const tokenText = token.raw;
+
+    // If adding this token would exceed the limit
+    if ((currentChunk + tokenText).length > maxLength) {
+      // Save current chunk if it has content
+      if (currentChunk) {
+        chunks.push(currentChunk);
+        currentChunk = "";
+      }
+
+      // If the token itself is too large, we have to split it
+      if (tokenText.length > maxLength) {
+        // Fall back to character-based splitting for this oversized block
+        const splits = characterSplit(tokenText, maxLength);
+        chunks.push(...splits.slice(0, -1));
+        currentChunk = splits[splits.length - 1];
+      } else {
+        // Token fits in a new chunk
+        currentChunk = tokenText;
+      }
+    } else {
+      // Token fits in current chunk
+      currentChunk += tokenText;
+    }
+  }
+
+  // Add any remaining content
+  if (currentChunk) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}
+
+/**
+ * Character-based splitting with word boundary awareness.
+ * Used as a fallback for oversized markdown blocks.
+ *
+ * @param text - The text to split
+ * @param maxLength - Maximum length per chunk
+ * @returns Array of text chunks
+ */
+function characterSplit(text: string, maxLength: number): string[] {
+  if (text.length <= maxLength) {
+    return [text];
+  }
+
+  const chunks: string[] = [];
+  let remaining = text;
+
+  while (remaining.length > maxLength) {
+    let splitIndex = maxLength;
+    const spaceIndex = remaining.lastIndexOf(" ", maxLength);
+
+    // If there's a space in the last 20% of the chunk, split there
+    if (spaceIndex > maxLength * 0.8) {
+      splitIndex = spaceIndex;
+    }
+
+    chunks.push(remaining.slice(0, splitIndex));
+    remaining = remaining.slice(splitIndex).trimStart();
+  }
+
+  if (remaining) {
+    chunks.push(remaining);
+  }
+
+  return chunks;
+}