ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter6/live-audio/backend/utils/textProcessor.js

@@ -0,0 +1,195 @@
+const emojiRegex = require('emoji-regex');
+
+// Remove emoji from the sentence
+function removeEmoji(sentence) {
+  const regex = emojiRegex();
+  return sentence.replace(regex, ' ').trim();
+}
+
+// Convert markdown to plain text
+function markdownToText(markdown) {
+  let text = markdown;
+  
+  // Remove links, keeping only the link text
+  text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1');
+  
+  // Remove headers
+  text = text.replace(/^#+\s*/gm, '');
+  
+  // Remove bold and italic markers
+  // Keep snake_case identifiers; only strip markdown __bold__ markers.
+  text = text.replace(/\*\*/g, '').replace(/\*/g, '')
+    .replace(/__/g, '');
+  
+  // Remove blockquotes
+  text = text.replace(/^>\s*/gm, '');
+  
+  // Remove horizontal rules
+  text = text.replace(/[-*_]{3,}/g, '');
+  
+  // Remove list markers
+  text = text.replace(/^[-*+]\s*/gm, '');
+  
+  // Remove code block markers
+  text = text.replace(/```/g, '');
+  
+  return text.trim();
+}
+
+// Convert numbers to words
+function numberToWords(num) {
+  const ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine'];
+  const tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
+  const teens = ['ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
+  const scales = ['', 'thousand', 'million', 'billion', 'trillion', 'quadrillion'];
+
+  if (num === 0) return 'zero';
+  if (!Number.isFinite(num)) return String(num);
+
+  function convertGroup(n) {
+    let result = '';
+    
+    if (n >= 100) {
+      result += ones[Math.floor(n / 100)] + ' hundred ';
+      n %= 100;
+    }
+    
+    if (n >= 20) {
+      result += tens[Math.floor(n / 10)] + ' ';
+      n %= 10;
+      if (n > 0) {
+        result += ones[n] + ' ';
+      }
+    } else if (n >= 10) {
+      result += teens[n - 10] + ' ';
+    } else if (n > 0) {
+      result += ones[n] + ' ';
+    }
+    
+    return result;
+  }
+
+  let result = '';
+  let groupIndex = 0;
+  
+  while (num > 0) {
+    const group = num % 1000;
+    if (group !== 0) {
+      const scale = scales[groupIndex] || '';
+      result = convertGroup(group) + scale + ' ' + result;
+    }
+    num = Math.floor(num / 1000);
+    groupIndex++;
+  }
+  
+  return result.trim();
+}
+
+// Pronounce special characters
+function pronounceSpecialCharacters(text, isCodeBlock = false) {
+  const specialCharMap = {
+    '@': 'at',
+    '#': 'hash',
+    '$': 'dollar',
+    '%': 'percent',
+    '^': 'caret',
+    '&': 'ampersand',
+    '*': 'asterisk',
+    '_': 'underscore',
+    '=': 'equals',
+    '+': 'plus',
+    '[': 'left square bracket',
+    ']': 'right square bracket',
+    '{': 'left curly brace',
+    '}': 'right curly brace',
+    '|': 'vertical bar',
+    '\\': 'backslash',
+    '<': 'less than',
+    '>': 'greater than',
+    '/': 'slash',
+    '`': 'backtick',
+    '~': 'tilde',
+  };
+
+  const punctuationMap = {
+    '!': 'exclamation',
+    '.': 'dot',
+    ',': 'comma',
+    '?': 'question mark',
+    ';': 'semicolon',
+    ':': 'colon',
+    '"': 'double quote',
+    "'": 'single quote',
+    '-': 'minus',
+    '(': 'left parenthesis',
+    ')': 'right parenthesis',
+  };
+
+  let processedText = text;
+  
+  // Replace special characters
+  Object.entries(specialCharMap).forEach(([char, pronunciation]) => {
+    processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
+  });
+
+  // Replace punctuation if in code block
+  if (isCodeBlock) {
+    Object.entries(punctuationMap).forEach(([char, pronunciation]) => {
+      processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
+    });
+  }
+
+  return processedText;
+}
+
+// Pronounce numbers in text
+function pronounceNumbers(text, language) {
+  if (language.startsWith('zh')) return text;
+
+  // Only consume the '.' when it is a real decimal point (digits follow).
+  // The old /(\d+\.?\d*)/ also matched "42." at the end of a sentence, which
+  // ate the period and emitted a dangling "point".
+  return text.replace(/\d+(?:\.\d+)?/g, match => {
+    const num = parseFloat(match);
+    if (isNaN(num)) return match;
+    
+    if (match.includes('.')) {
+      const [integer, decimal] = match.split('.');
+      return `${numberToWords(parseInt(integer))} point ${decimal.split('').map(d => numberToWords(parseInt(d))).join(' ')}`;
+    }
+    return numberToWords(parseInt(match));
+  });
+}
+
+// Remove emotional indicators
+function removeEmotions(text) {
+  return text.replace(/\*[a-zA-Z0-9 -]*\*/g, '').trim();
+}
+
+// Process code blocks
+function pronounceCodeBlock(text) {
+  return text.replace(/`([^`\n]+)`|```(?:[\s\S]*?)```/g, (match) => {
+    const content = match.startsWith('```') 
+      ? match.slice(3, -3)
+      : match.slice(1, -1);
+    return pronounceSpecialCharacters(content, true);
+  });
+}
+
+// Main preprocessing function
+function preprocessSentence(sentence, language = 'en') {
+  let processed = sentence;
+  processed = pronounceCodeBlock(processed);
+  processed = markdownToText(processed);
+  processed = pronounceNumbers(processed, language);
+  processed = removeEmotions(processed);
+  processed = pronounceSpecialCharacters(processed);
+  processed = removeEmoji(processed);
+  return processed.trim();
+}
+
+module.exports = {
+  preprocessSentence,
+  numberToWords,
+  markdownToText,
+};