ai-agent-book/chapter6/live-audio/backend/utils/textProcessor.js

const emojiRegex = require('emoji-regex');

// Remove emoji from the sentence
function removeEmoji(sentence) {
  const regex = emojiRegex();
  return sentence.replace(regex, ' ').trim();
}

// Convert markdown to plain text
function markdownToText(markdown) {
  let text = markdown;
  
  // Remove links, keeping only the link text
  text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1');
  
  // Remove headers
  text = text.replace(/^#+\s*/gm, '');
  
  // Remove bold and italic markers
  // Keep snake_case identifiers; only strip markdown __bold__ markers.
  text = text.replace(/\*\*/g, '').replace(/\*/g, '')
    .replace(/__/g, '');
  
  // Remove blockquotes
  text = text.replace(/^>\s*/gm, '');
  
  // Remove horizontal rules
  text = text.replace(/[-*_]{3,}/g, '');
  
  // Remove list markers
  text = text.replace(/^[-*+]\s*/gm, '');
  
  // Remove code block markers
  text = text.replace(/```/g, '');
  
  return text.trim();
}

// Convert numbers to words
function numberToWords(num) {
  const ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine'];
  const tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
  const teens = ['ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
  const scales = ['', 'thousand', 'million', 'billion', 'trillion', 'quadrillion'];

  if (num === 0) return 'zero';
  if (!Number.isFinite(num)) return String(num);

  function convertGroup(n) {
    let result = '';
    
    if (n >= 100) {
      result += ones[Math.floor(n / 100)] + ' hundred ';
      n %= 100;
    }
    
    if (n >= 20) {
      result += tens[Math.floor(n / 10)] + ' ';
      n %= 10;
      if (n > 0) {
        result += ones[n] + ' ';
      }
    } else if (n >= 10) {
      result += teens[n - 10] + ' ';
    } else if (n > 0) {
      result += ones[n] + ' ';
    }
    
    return result;
  }

  let result = '';
  let groupIndex = 0;
  
  while (num > 0) {
    const group = num % 1000;
    if (group !== 0) {
      const scale = scales[groupIndex] || '';
      result = convertGroup(group) + scale + ' ' + result;
    }
    num = Math.floor(num / 1000);
    groupIndex++;
  }
  
  return result.trim();
}

// Pronounce special characters
function pronounceSpecialCharacters(text, isCodeBlock = false) {
  const specialCharMap = {
    '@': 'at',
    '#': 'hash',
    '$': 'dollar',
    '%': 'percent',
    '^': 'caret',
    '&': 'ampersand',
    '*': 'asterisk',
    '_': 'underscore',
    '=': 'equals',
    '+': 'plus',
    '[': 'left square bracket',
    ']': 'right square bracket',
    '{': 'left curly brace',
    '}': 'right curly brace',
    '|': 'vertical bar',
    '\\': 'backslash',
    '<': 'less than',
    '>': 'greater than',
    '/': 'slash',
    '`': 'backtick',
    '~': 'tilde',
  };

  const punctuationMap = {
    '!': 'exclamation',
    '.': 'dot',
    ',': 'comma',
    '?': 'question mark',
    ';': 'semicolon',
    ':': 'colon',
    '"': 'double quote',
    "'": 'single quote',
    '-': 'minus',
    '(': 'left parenthesis',
    ')': 'right parenthesis',
  };

  let processedText = text;
  
  // Replace special characters
  Object.entries(specialCharMap).forEach(([char, pronunciation]) => {
    processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
  });

  // Replace punctuation if in code block
  if (isCodeBlock) {
    Object.entries(punctuationMap).forEach(([char, pronunciation]) => {
      processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
    });
  }

  return processedText;
}

// Pronounce numbers in text
function pronounceNumbers(text, language) {
  if (language.startsWith('zh')) return text;

  // Only consume the '.' when it is a real decimal point (digits follow).
  // The old /(\d+\.?\d*)/ also matched "42." at the end of a sentence, which
  // ate the period and emitted a dangling "point".
  return text.replace(/\d+(?:\.\d+)?/g, match => {
    const num = parseFloat(match);
    if (isNaN(num)) return match;
    
    if (match.includes('.')) {
      const [integer, decimal] = match.split('.');
      return `${numberToWords(parseInt(integer))} point ${decimal.split('').map(d => numberToWords(parseInt(d))).join(' ')}`;
    }
    return numberToWords(parseInt(match));
  });
}

// Remove emotional indicators
function removeEmotions(text) {
  return text.replace(/\*[a-zA-Z0-9 -]*\*/g, '').trim();
}

// Process code blocks
function pronounceCodeBlock(text) {
  return text.replace(/`([^`\n]+)`|```(?:[\s\S]*?)```/g, (match) => {
    const content = match.startsWith('```') 
      ? match.slice(3, -3)
      : match.slice(1, -1);
    return pronounceSpecialCharacters(content, true);
  });
}

// Main preprocessing function
function preprocessSentence(sentence, language = 'en') {
  let processed = sentence;
  processed = pronounceCodeBlock(processed);
  processed = markdownToText(processed);
  processed = pronounceNumbers(processed, language);
  processed = removeEmotions(processed);
  processed = pronounceSpecialCharacters(processed);
  processed = removeEmoji(processed);
  return processed.trim();
}

module.exports = {
  preprocessSentence,
  numberToWords,
  markdownToText,
};