ai-agent-book 精选快照（<2MB 代码与文档，来自 github.com/bojieli/ai-agent-book）

chapter6/live-audio/backend/utils/providers/asrProviders.js

@@ -0,0 +1,276 @@
+const axios = require('axios');
+const FormData = require('form-data');
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * Base ASR Provider class
+ */
+class BaseASRProvider {
+  constructor(config, apiKey) {
+    this.config = config;
+    this.apiKey = apiKey;
+  }
+
+  /**
+   * Convert raw audio buffer to WAV format
+   * @param {Buffer} audioBuffer - Raw PCM audio data
+   * @param {Object} options - Audio format options
+   * @returns {Buffer} WAV formatted audio data
+   */
+  createWavBuffer(audioBuffer, options = {}) {
+    const sampleRate = options.sampleRate || 16000;
+    const channels = options.channels || 1;
+    const bitsPerSample = options.bitsPerSample || 16;
+    
+    const byteRate = sampleRate * channels * bitsPerSample / 8;
+    const blockAlign = channels * bitsPerSample / 8;
+    const dataSize = audioBuffer.length;
+    const fileSize = 36 + dataSize;
+    
+    const header = Buffer.alloc(44);
+    
+    // RIFF header
+    header.write('RIFF', 0);
+    header.writeUInt32LE(fileSize, 4);
+    header.write('WAVE', 8);
+    
+    // fmt chunk
+    header.write('fmt ', 12);
+    header.writeUInt32LE(16, 16); // PCM format chunk size
+    header.writeUInt16LE(1, 20);  // PCM format
+    header.writeUInt16LE(channels, 22);
+    header.writeUInt32LE(sampleRate, 24);
+    header.writeUInt32LE(byteRate, 28);
+    header.writeUInt16LE(blockAlign, 32);
+    header.writeUInt16LE(bitsPerSample, 34);
+    
+    // data chunk
+    header.write('data', 36);
+    header.writeUInt32LE(dataSize, 40);
+    
+    return Buffer.concat([header, audioBuffer]);
+  }
+
+  async transcribe(audioBuffer, options = {}) {
+    throw new Error('transcribe method must be implemented by subclass');
+  }
+}
+
+/**
+ * OpenAI Whisper ASR Provider
+ */
+class OpenAIASRProvider extends BaseASRProvider {
+  async transcribe(audioBuffer, tempDir, options = {}) {
+    try {
+      // Create WAV buffer
+      const wavBuffer = this.createWavBuffer(audioBuffer, {
+        sampleRate: 16000,
+        channels: 1,
+        bitsPerSample: 16
+      });
+
+      // Create temporary file
+      const tempFileName = `audio_${Date.now()}_${Math.random().toString(36).substring(2)}.wav`;
+      const tempFilePath = path.join(tempDir, tempFileName);
+      
+      // Write audio to temporary file
+      fs.writeFileSync(tempFilePath, wavBuffer);
+
+      try {
+        // Create form data
+        const formData = new FormData();
+        formData.append('file', fs.createReadStream(tempFilePath));
+        formData.append('model', this.config.model);
+        formData.append('response_format', 'json');
+        
+        if (options.language) {
+          formData.append('language', options.language);
+        }
+        
+        if (options.prompt) {
+          formData.append('prompt', options.prompt);
+        }
+
+        // Make API request
+        const response = await axios({
+          method: 'post',
+          url: this.config.apiUrl,
+          data: formData,
+          headers: {
+            'Authorization': `Bearer ${this.apiKey}`,
+            ...formData.getHeaders()
+          },
+          timeout: 30000 // 30 second timeout
+        });
+
+        const result = {
+          success: true,
+          text: response.data.text || '',
+          language: response.data.language || 'unknown',
+          duration: response.data.duration || 0,
+          confidence: response.data.confidence || 1.0,
+          requestId: response.headers?.['x-request-id'] || response.headers?.['request-id'] || null,
+          responseModel: response.data.model || this.config.model,
+          timestamp: Date.now(),
+          provider: 'openai'
+        };
+
+        console.log('OpenAI ASR Result:', {
+          text: result.text,
+          language: result.language,
+          duration: result.duration
+        });
+
+        return result;
+
+      } finally {
+        // Clean up temporary file
+        try {
+          fs.unlinkSync(tempFilePath);
+        } catch (cleanupError) {
+          console.warn('Failed to cleanup temp file:', cleanupError.message);
+        }
+      }
+
+    } catch (error) {
+      console.error('OpenAI ASR error:', error.response?.data || error.message);
+      
+      return {
+        success: false,
+        text: '',
+        error: error.response?.data?.error?.message || error.message,
+        timestamp: Date.now(),
+        provider: 'openai'
+      };
+    }
+  }
+}
+
+/**
+ * SenseVoice via Siliconflow ASR Provider
+ */
+class SiliconflowASRProvider extends BaseASRProvider {
+  async transcribe(audioBuffer, tempDir, options = {}) {
+    try {
+      // Create WAV buffer
+      const wavBuffer = this.createWavBuffer(audioBuffer, {
+        sampleRate: 16000,
+        channels: 1,
+        bitsPerSample: 16
+      });
+
+      // Create temporary file
+      const tempFileName = `audio_${Date.now()}_${Math.random().toString(36).substring(2)}.wav`;
+      const tempFilePath = path.join(tempDir, tempFileName);
+      
+      // Write audio to temporary file
+      fs.writeFileSync(tempFilePath, wavBuffer);
+
+      try {
+        // Create form data
+        const formData = new FormData();
+        formData.append('file', fs.createReadStream(tempFilePath));
+        formData.append('model', this.config.model);
+        formData.append('response_format', 'json');
+        
+        // SenseVoice specific parameters
+        if (options.language) {
+          formData.append('language', options.language);
+        } else {
+          // SenseVoice supports auto language detection
+          formData.append('language', 'auto');
+        }
+        
+        if (options.prompt) {
+          formData.append('prompt', options.prompt);
+        }
+
+        // Make API request
+        const response = await axios({
+          method: 'post',
+          url: this.config.apiUrl,
+          data: formData,
+          headers: {
+            'Authorization': `Bearer ${this.apiKey}`,
+            ...formData.getHeaders()
+          },
+          timeout: 30000 // 30 second timeout
+        });
+
+        const result = {
+          success: true,
+          text: response.data.text || '',
+          language: response.data.language || 'unknown',
+          duration: response.data.duration || 0,
+          confidence: response.data.confidence || 1.0,
+          requestId: response.headers?.['x-request-id'] || response.headers?.['request-id'] || null,
+          responseModel: response.data.model || this.config.model,
+          timestamp: Date.now(),
+          provider: 'siliconflow'
+        };
+
+        console.log('SenseVoice ASR Result:', {
+          text: result.text,
+          language: result.language,
+          duration: result.duration
+        });
+
+        return result;
+
+      } finally {
+        // Clean up temporary file
+        try {
+          fs.unlinkSync(tempFilePath);
+        } catch (cleanupError) {
+          console.warn('Failed to cleanup temp file:', cleanupError.message);
+        }
+      }
+
+    } catch (error) {
+      console.error('SenseVoice ASR error:', error.response?.data || error.message);
+      
+      return {
+        success: false,
+        text: '',
+        error: error.response?.data?.error?.message || error.message,
+        timestamp: Date.now(),
+        provider: 'siliconflow'
+      };
+    }
+  }
+}
+
+/**
+ * ASR Provider Factory
+ */
+class ASRProviderFactory {
+  static createProvider(providerName, config, globalConfig) {
+    const providerConfig = config.ASR_PROVIDERS[providerName];
+    if (!providerConfig) {
+      throw new Error(`ASR provider ${providerName} not found in configuration`);
+    }
+
+    // Get API key from global config
+    const apiKey = globalConfig[providerConfig.apiKey];
+    if (!apiKey) {
+      throw new Error(`API key ${providerConfig.apiKey} not found in configuration`);
+    }
+
+    switch (providerName) {
+      case 'openai':
+        return new OpenAIASRProvider(providerConfig, apiKey);
+      case 'siliconflow':
+        return new SiliconflowASRProvider(providerConfig, apiKey);
+      default:
+        throw new Error(`Unsupported ASR provider: ${providerName}`);
+    }
+  }
+}
+
+module.exports = {
+  BaseASRProvider,
+  OpenAIASRProvider,
+  SiliconflowASRProvider,
+  ASRProviderFactory
+};

chapter6/live-audio/backend/utils/providers/llmProviders.js

@@ -0,0 +1,248 @@
+const axios = require('axios');
+
+function providerErrorMessage(error) {
+  const data = error.response?.data;
+  if (data && typeof data === 'object' && !data.readable) {
+    return data.error?.message || data.message || error.message;
+  }
+  return error.response?.status
+    ? `HTTP ${error.response.status}: ${error.response.statusText || error.message}`
+    : error.message;
+}
+
+/**
+ * Base LLM Provider class
+ */
+class BaseLLMProvider {
+  constructor(config, apiKey) {
+    this.config = config;
+    this.apiKey = apiKey;
+  }
+
+  async createChatCompletion(messages, options = {}) {
+    throw new Error('createChatCompletion method must be implemented by subclass');
+  }
+
+  /**
+   * Prepare headers for API request
+   * @returns {Object} Headers object
+   */
+  getHeaders() {
+    return {
+      'Authorization': `Bearer ${this.apiKey}`,
+      'Content-Type': 'application/json'
+    };
+  }
+
+  /**
+   * Prepare request payload
+   * @param {Array} messages - Chat messages
+   * @param {Object} options - Additional options
+   * @returns {Object} Request payload
+   */
+  getRequestPayload(messages, options = {}) {
+    return {
+      model: this.config.model,
+      messages: messages,
+      stream: true,
+      max_tokens: options.max_tokens || 4096,
+      ...options
+    };
+  }
+}
+
+/**
+ * OpenAI LLM Provider
+ */
+class OpenAILLMProvider extends BaseLLMProvider {
+  async createChatCompletion(messages, options = {}) {
+    try {
+      const payload = this.getRequestPayload(messages, options);
+      const headers = this.getHeaders();
+
+      console.log('OpenAI LLM Request:', {
+        model: payload.model,
+        messagesCount: messages.length,
+        stream: payload.stream
+      });
+
+      const response = await axios.post(
+        this.config.apiUrl,
+        payload,
+        {
+          headers: headers,
+          cancelToken: options.cancelToken,
+          responseType: 'stream'
+        }
+      );
+
+      return {
+        success: true,
+        response: response,
+        provider: 'openai'
+      };
+
+    } catch (error) {
+      const message = providerErrorMessage(error);
+      console.error('OpenAI LLM error:', message);
+      return {
+        success: false,
+        error: message,
+        provider: 'openai'
+      };
+    }
+  }
+}
+
+/**
+ * OpenRouter LLM Provider (supports both GPT-4o and Gemini)
+ */
+class OpenRouterLLMProvider extends BaseLLMProvider {
+  getHeaders() {
+    return {
+      'Authorization': `Bearer ${this.apiKey}`,
+      'Content-Type': 'application/json',
+      'HTTP-Referer': 'https://live-audio-chat.local',
+      'X-Title': 'Live Audio Chat'
+    };
+  }
+
+  async createChatCompletion(messages, options = {}) {
+    try {
+      const payload = this.getRequestPayload(messages, options);
+      const headers = this.getHeaders();
+
+      console.log('OpenRouter LLM Request:', {
+        model: payload.model,
+        messagesCount: messages.length,
+        stream: payload.stream
+      });
+
+      const response = await axios.post(
+        this.config.apiUrl,
+        payload,
+        {
+          headers: headers,
+          cancelToken: options.cancelToken,
+          responseType: 'stream'
+        }
+      );
+
+      return {
+        success: true,
+        response: response,
+        provider: 'openrouter'
+      };
+
+    } catch (error) {
+      const message = providerErrorMessage(error);
+      console.error('OpenRouter LLM error:', message);
+      return {
+        success: false,
+        error: message,
+        provider: 'openrouter'
+      };
+    }
+  }
+}
+
+/**
+ * ARK (Doubao) LLM Provider
+ */
+class ARKLLMProvider extends BaseLLMProvider {
+  getHeaders() {
+    return {
+      'Authorization': `Bearer ${this.apiKey}`,
+      'Content-Type': 'application/json'
+    };
+  }
+
+  getRequestPayload(messages, options = {}) {
+    // ARK API format is similar to OpenAI but may have slight differences
+    return {
+      model: this.config.model,
+      messages: messages,
+      stream: true,
+      max_tokens: options.max_tokens || 4096,
+      temperature: options.temperature || 0.7,
+      ...options
+    };
+  }
+
+  async createChatCompletion(messages, options = {}) {
+    try {
+      const payload = this.getRequestPayload(messages, options);
+      const headers = this.getHeaders();
+
+      console.log('ARK (Doubao) LLM Request:', {
+        model: payload.model,
+        messagesCount: messages.length,
+        stream: payload.stream
+      });
+
+      const response = await axios.post(
+        this.config.apiUrl,
+        payload,
+        {
+          headers: headers,
+          cancelToken: options.cancelToken,
+          responseType: 'stream'
+        }
+      );
+
+      return {
+        success: true,
+        response: response,
+        provider: 'ark'
+      };
+
+    } catch (error) {
+      const message = providerErrorMessage(error);
+      console.error('ARK (Doubao) LLM error:', message);
+      return {
+        success: false,
+        error: message,
+        provider: 'ark'
+      };
+    }
+  }
+}
+
+/**
+ * LLM Provider Factory
+ */
+class LLMProviderFactory {
+  static createProvider(providerName, config, globalConfig) {
+    const providerConfig = config.LLM_PROVIDERS[providerName];
+    if (!providerConfig) {
+      throw new Error(`LLM provider ${providerName} not found in configuration`);
+    }
+
+    // Get API key from global config
+    const apiKey = globalConfig[providerConfig.apiKey];
+    if (!apiKey) {
+      throw new Error(`API key ${providerConfig.apiKey} not found in configuration`);
+    }
+
+    switch (providerName) {
+      case 'openai':
+        return new OpenAILLMProvider(providerConfig, apiKey);
+      case 'openrouter':
+      case 'openrouter-gpt':
+      case 'openrouter-gemini':
+        return new OpenRouterLLMProvider(providerConfig, apiKey);
+      case 'ark':
+        return new ARKLLMProvider(providerConfig, apiKey);
+      default:
+        throw new Error(`Unsupported LLM provider: ${providerName}`);
+    }
+  }
+}
+
+module.exports = {
+  BaseLLMProvider,
+  OpenAILLMProvider,
+  OpenRouterLLMProvider,
+  ARKLLMProvider,
+  LLMProviderFactory
+};

chapter6/live-audio/backend/utils/speechToText.js

@@ -0,0 +1,147 @@
+const fs = require('fs');
+const path = require('path');
+const config = require('../config');
+const { ASRProviderFactory } = require('./providers/asrProviders');
+
+class SpeechToTextService {
+  constructor() {
+    this.tempDir = path.join(__dirname, '../temp');
+    this.ensureTempDirectory();
+    
+    // Initialize ASR provider based on configuration
+    this.initializeProvider();
+  }
+
+  /**
+   * Initialize ASR provider based on configuration
+   */
+  initializeProvider() {
+    try {
+      const providerName = config.ASR_PROVIDER || 'openai';
+      this.asrProvider = ASRProviderFactory.createProvider(providerName, config, config);
+      console.log(`ASR Provider initialized: ${providerName}`);
+    } catch (error) {
+      console.error('Failed to initialize ASR provider:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Switch ASR provider dynamically
+   * @param {string} providerName - Provider name to switch to
+   */
+  switchProvider(providerName) {
+    try {
+      this.asrProvider = ASRProviderFactory.createProvider(providerName, config, config);
+      console.log(`ASR Provider switched to: ${providerName}`);
+    } catch (error) {
+      console.error('Failed to switch ASR provider:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Ensure temp directory exists
+   */
+  ensureTempDirectory() {
+    if (!fs.existsSync(this.tempDir)) {
+      fs.mkdirSync(this.tempDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Transcribe audio using the configured ASR provider
+   * @param {Buffer} audioBuffer - Raw audio data
+   * @param {Object} options - Transcription options
+   * @returns {Promise<Object>} Transcription result
+   */
+  async transcribeAudio(audioBuffer, options = {}) {
+    if (!this.asrProvider) {
+      throw new Error('ASR provider not initialized');
+    }
+
+    try {
+      const result = await this.asrProvider.transcribe(audioBuffer, this.tempDir, options);
+      return result;
+    } catch (error) {
+      console.error('Transcription error:', error);
+      return {
+        success: false,
+        text: '',
+        error: error.message,
+        timestamp: Date.now(),
+        provider: this.asrProvider.config ? 'unknown' : 'uninitialized'
+      };
+    }
+  }
+
+  /**
+   * Clean up old temporary files
+   */
+  cleanupTempFiles() {
+    try {
+      const files = fs.readdirSync(this.tempDir);
+      const now = Date.now();
+      const maxAge = 10 * 60 * 1000; // 10 minutes
+      
+      files.forEach(file => {
+        const filePath = path.join(this.tempDir, file);
+        const stats = fs.statSync(filePath);
+        
+        if (now - stats.mtime.getTime() > maxAge) {
+          try {
+            fs.unlinkSync(filePath);
+            console.log('Cleaned up old temp file:', file);
+          } catch (error) {
+            console.warn('Failed to cleanup old temp file:', file, error.message);
+          }
+        }
+      });
+    } catch (error) {
+      console.warn('Failed to cleanup temp directory:', error.message);
+    }
+  }
+
+  /**
+   * Check if audio buffer has sufficient content for transcription
+   * @param {Buffer} audioBuffer - Audio buffer to check
+   * @returns {boolean} True if buffer has sufficient content
+   */
+  hasSufficientAudio(audioBuffer) {
+    if (!audioBuffer || audioBuffer.length === 0) {
+      return false;
+    }
+
+    // Check minimum duration (at least 0.1 seconds of audio)
+    const minSamples = config.AUDIO_SAMPLE_RATE * 0.1; // 0.1 seconds
+    const minBytes = minSamples * 2; // 16-bit = 2 bytes per sample
+    
+    if (audioBuffer.length < minBytes) {
+      return false;
+    }
+
+    // Since we're using Silero VAD for speech detection, we trust its decision
+    // and only check for minimum duration. The energy check is redundant and
+    // can reject valid speech that Silero VAD correctly identified.
+    return true;
+  }
+
+  /**
+   * Get current provider information
+   * @returns {Object} Current provider information
+   */
+  getProviderInfo() {
+    if (!this.asrProvider) {
+      return { provider: 'none', status: 'not initialized' };
+    }
+
+    return {
+      provider: this.asrProvider.constructor.name,
+      model: this.asrProvider.config.model,
+      apiUrl: this.asrProvider.config.apiUrl,
+      status: 'ready'
+    };
+  }
+}
+
+module.exports = SpeechToTextService; 

chapter6/live-audio/backend/utils/test_markdown_snake_case.js

@@ -0,0 +1,12 @@
+const assert = require('assert');
+const Module = require('module');
+const orig = Module.prototype.require;
+Module.prototype.require = function (id) {
+  if (id === 'emoji-regex') return () => /(?!)/g;
+  return orig.apply(this, arguments);
+};
+const { markdownToText } = require('./textProcessor.js');
+const out = markdownToText('Call get_user_id then save_to_db');
+assert.ok(out.includes('get_user_id'));
+assert.ok(out.includes('save_to_db'));
+console.log('ok');

chapter6/live-audio/backend/utils/test_number_to_words.js

@@ -0,0 +1,13 @@
+const assert = require('assert');
+const Module = require('module');
+const orig = Module.prototype.require;
+Module.prototype.require = function (id) {
+  if (id === 'emoji-regex') return () => /(?!)/g;
+  return orig.apply(this, arguments);
+};
+const { numberToWords } = require('./textProcessor.js');
+assert.strictEqual(numberToWords(0), 'zero');
+assert.ok(!numberToWords(1e12).includes('undefined'));
+assert.ok(numberToWords(1e12).includes('trillion'));
+assert.strictEqual(numberToWords(Infinity), 'Infinity');
+console.log('ok');

chapter6/live-audio/backend/utils/textProcessor.js

@@ -0,0 +1,195 @@
+const emojiRegex = require('emoji-regex');
+
+// Remove emoji from the sentence
+function removeEmoji(sentence) {
+  const regex = emojiRegex();
+  return sentence.replace(regex, ' ').trim();
+}
+
+// Convert markdown to plain text
+function markdownToText(markdown) {
+  let text = markdown;
+  
+  // Remove links, keeping only the link text
+  text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1');
+  
+  // Remove headers
+  text = text.replace(/^#+\s*/gm, '');
+  
+  // Remove bold and italic markers
+  // Keep snake_case identifiers; only strip markdown __bold__ markers.
+  text = text.replace(/\*\*/g, '').replace(/\*/g, '')
+    .replace(/__/g, '');
+  
+  // Remove blockquotes
+  text = text.replace(/^>\s*/gm, '');
+  
+  // Remove horizontal rules
+  text = text.replace(/[-*_]{3,}/g, '');
+  
+  // Remove list markers
+  text = text.replace(/^[-*+]\s*/gm, '');
+  
+  // Remove code block markers
+  text = text.replace(/```/g, '');
+  
+  return text.trim();
+}
+
+// Convert numbers to words
+function numberToWords(num) {
+  const ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine'];
+  const tens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'];
+  const teens = ['ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'];
+  const scales = ['', 'thousand', 'million', 'billion', 'trillion', 'quadrillion'];
+
+  if (num === 0) return 'zero';
+  if (!Number.isFinite(num)) return String(num);
+
+  function convertGroup(n) {
+    let result = '';
+    
+    if (n >= 100) {
+      result += ones[Math.floor(n / 100)] + ' hundred ';
+      n %= 100;
+    }
+    
+    if (n >= 20) {
+      result += tens[Math.floor(n / 10)] + ' ';
+      n %= 10;
+      if (n > 0) {
+        result += ones[n] + ' ';
+      }
+    } else if (n >= 10) {
+      result += teens[n - 10] + ' ';
+    } else if (n > 0) {
+      result += ones[n] + ' ';
+    }
+    
+    return result;
+  }
+
+  let result = '';
+  let groupIndex = 0;
+  
+  while (num > 0) {
+    const group = num % 1000;
+    if (group !== 0) {
+      const scale = scales[groupIndex] || '';
+      result = convertGroup(group) + scale + ' ' + result;
+    }
+    num = Math.floor(num / 1000);
+    groupIndex++;
+  }
+  
+  return result.trim();
+}
+
+// Pronounce special characters
+function pronounceSpecialCharacters(text, isCodeBlock = false) {
+  const specialCharMap = {
+    '@': 'at',
+    '#': 'hash',
+    '$': 'dollar',
+    '%': 'percent',
+    '^': 'caret',
+    '&': 'ampersand',
+    '*': 'asterisk',
+    '_': 'underscore',
+    '=': 'equals',
+    '+': 'plus',
+    '[': 'left square bracket',
+    ']': 'right square bracket',
+    '{': 'left curly brace',
+    '}': 'right curly brace',
+    '|': 'vertical bar',
+    '\\': 'backslash',
+    '<': 'less than',
+    '>': 'greater than',
+    '/': 'slash',
+    '`': 'backtick',
+    '~': 'tilde',
+  };
+
+  const punctuationMap = {
+    '!': 'exclamation',
+    '.': 'dot',
+    ',': 'comma',
+    '?': 'question mark',
+    ';': 'semicolon',
+    ':': 'colon',
+    '"': 'double quote',
+    "'": 'single quote',
+    '-': 'minus',
+    '(': 'left parenthesis',
+    ')': 'right parenthesis',
+  };
+
+  let processedText = text;
+  
+  // Replace special characters
+  Object.entries(specialCharMap).forEach(([char, pronunciation]) => {
+    processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
+  });
+
+  // Replace punctuation if in code block
+  if (isCodeBlock) {
+    Object.entries(punctuationMap).forEach(([char, pronunciation]) => {
+      processedText = processedText.replace(new RegExp(char.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), ` ${pronunciation} `);
+    });
+  }
+
+  return processedText;
+}
+
+// Pronounce numbers in text
+function pronounceNumbers(text, language) {
+  if (language.startsWith('zh')) return text;
+
+  // Only consume the '.' when it is a real decimal point (digits follow).
+  // The old /(\d+\.?\d*)/ also matched "42." at the end of a sentence, which
+  // ate the period and emitted a dangling "point".
+  return text.replace(/\d+(?:\.\d+)?/g, match => {
+    const num = parseFloat(match);
+    if (isNaN(num)) return match;
+    
+    if (match.includes('.')) {
+      const [integer, decimal] = match.split('.');
+      return `${numberToWords(parseInt(integer))} point ${decimal.split('').map(d => numberToWords(parseInt(d))).join(' ')}`;
+    }
+    return numberToWords(parseInt(match));
+  });
+}
+
+// Remove emotional indicators
+function removeEmotions(text) {
+  return text.replace(/\*[a-zA-Z0-9 -]*\*/g, '').trim();
+}
+
+// Process code blocks
+function pronounceCodeBlock(text) {
+  return text.replace(/`([^`\n]+)`|```(?:[\s\S]*?)```/g, (match) => {
+    const content = match.startsWith('```') 
+      ? match.slice(3, -3)
+      : match.slice(1, -1);
+    return pronounceSpecialCharacters(content, true);
+  });
+}
+
+// Main preprocessing function
+function preprocessSentence(sentence, language = 'en') {
+  let processed = sentence;
+  processed = pronounceCodeBlock(processed);
+  processed = markdownToText(processed);
+  processed = pronounceNumbers(processed, language);
+  processed = removeEmotions(processed);
+  processed = pronounceSpecialCharacters(processed);
+  processed = removeEmoji(processed);
+  return processed.trim();
+}
+
+module.exports = {
+  preprocessSentence,
+  numberToWords,
+  markdownToText,
+};

chapter6/live-audio/backend/utils/vad.js

@@ -0,0 +1,306 @@
+const config = require('../config');
+const ort = require('onnxruntime-node');
+const path = require('path');
+const fs = require('fs');
+
+class VoiceActivityDetector {
+  constructor(options = {}) {
+    this.threshold = options.threshold || config.VAD_THRESHOLD || 0.5;
+    this.frameLength = options.frameLength || config.VAD_FRAME_LENGTH || 512;
+    this.minSpeechDuration = options.minSpeechDuration || config.VAD_MIN_SPEECH_DURATION || 250;
+    this.maxSilenceDuration = options.maxSilenceDuration || config.VAD_MAX_SILENCE_DURATION || 500;
+    this.sampleRate = options.sampleRate || config.AUDIO_SAMPLE_RATE || 16000;
+    
+    // Silero VAD specific parameters
+    this.sileroFrameLength = 512; // Fixed frame length for Silero VAD
+    this.sileroSampleRate = 16000; // Fixed sample rate for Silero VAD
+    
+    // State variables
+    this.isSpeaking = false;
+    this.speechStartTime = null;
+    this.lastSpeechTime = null;
+    this.audioBuffer = Buffer.alloc(0);
+    this.speechBuffer = Buffer.alloc(0);
+    
+    // ONNX Runtime session
+    this.session = null;
+    this.isInitialized = false;
+    this.initializationPromise = null;
+    
+    // Silero VAD state
+    this.state = null;
+    this.sr = null;
+    
+    // Initialize the Silero VAD
+    this.initializationPromise = this.initializeSileroVAD();
+  }
+
+  /**
+   * Initialize the Silero VAD ONNX model
+   */
+  async initializeSileroVAD() {
+    try {
+      console.log('Initializing Silero VAD...');
+      
+      // Load the ONNX model
+      const modelPath = path.join(__dirname, '../models/silero_vad.onnx');
+      
+      if (!fs.existsSync(modelPath)) {
+        throw new Error(`Silero VAD model not found at ${modelPath}`);
+      }
+      
+      // Create ONNX Runtime session
+      this.session = await ort.InferenceSession.create(modelPath, {
+        executionProviders: ['cpu'],
+        graphOptimizationLevel: 'all',
+        enableMemPattern: true
+      });
+      
+      // Initialize state tensors for Silero VAD
+      this.resetSileroState();
+      
+      this.isInitialized = true;
+      console.log('Silero VAD initialized successfully');
+      
+    } catch (error) {
+      console.error('Failed to initialize Silero VAD:', error);
+      throw error;
+    }
+  }
+
+  /**
+   * Reset Silero VAD state tensors
+   */
+  resetSileroState() {
+    // Initialize state tensor for Silero VAD
+    // The state tensor combines h and c LSTM hidden states
+    this.state = new ort.Tensor('float32', new Float32Array(2 * 1 * 128).fill(0.0), [2, 1, 128]);
+    this.sr = new ort.Tensor('int64', new BigInt64Array([BigInt(this.sileroSampleRate)]), [1]);
+  }
+
+  /**
+   * Convert Buffer to Float32Array for Silero VAD
+   * @param {Buffer} buffer - PCM 16-bit audio buffer
+   * @returns {Float32Array} Normalized audio samples
+   */
+  bufferToFloat32Array(buffer) {
+    const samples = new Float32Array(buffer.length / 2);
+    for (let i = 0; i < samples.length; i++) {
+      // Convert 16-bit PCM to normalized float (-1 to 1)
+      const sample = buffer.readInt16LE(i * 2);
+      samples[i] = sample / 32768.0;
+    }
+    return samples;
+  }
+
+  /**
+   * Run Silero VAD inference on audio chunk
+   * @param {Float32Array} audioSamples - Normalized audio samples
+   * @returns {Promise<number>} Speech probability (0-1)
+   */
+  async runSileroVAD(audioSamples) {
+    if (!this.session || !this.isInitialized) {
+      throw new Error('Silero VAD not initialized');
+    }
+
+    try {
+      // Create input tensor
+      const inputTensor = new ort.Tensor('float32', audioSamples, [1, audioSamples.length]);
+      
+      // Run inference
+      const feeds = {
+        input: inputTensor,
+        state: this.state,
+        sr: this.sr
+      };
+      
+      const results = await this.session.run(feeds);
+      
+      // Update state tensor for next inference
+      this.state = results.stateN;
+      
+      // Get speech probability
+      const speechProb = results.output.data[0];
+      
+      return speechProb;
+      
+    } catch (error) {
+      console.error('Error running Silero VAD inference:', error);
+      return 0.0; // Return silence probability on error
+    }
+  }
+
+  /**
+   * Process audio chunk and detect voice activity using Silero VAD
+   * @param {Buffer} audioChunk - Raw audio data
+   * @returns {Promise<Array>} VAD result with detection status and audio data
+   */
+  async processAudioChunk(audioChunk) {
+    // Wait for initialization if not complete
+    if (!this.isInitialized) {
+      await this.initializationPromise;
+    }
+
+    const currentTime = Date.now();
+    const results = [];
+    
+    // Add to buffer
+    this.audioBuffer = Buffer.concat([this.audioBuffer, audioChunk]);
+    
+    // Process audio in chunks suitable for Silero VAD (512 samples)
+    const chunkSize = this.sileroFrameLength * 2; // 16-bit = 2 bytes per sample
+    
+    while (this.audioBuffer.length >= chunkSize) {
+      const chunk = this.audioBuffer.slice(0, chunkSize);
+      this.audioBuffer = this.audioBuffer.slice(chunkSize);
+      
+      try {
+        // Convert to Float32Array for Silero VAD
+        const audioSamples = this.bufferToFloat32Array(chunk);
+        
+        // Run Silero VAD inference
+        const speechProb = await this.runSileroVAD(audioSamples);
+        
+        // Check if speech is detected
+        const isVoiceActive = speechProb > this.threshold;
+        
+        if (isVoiceActive) {
+          if (!this.isSpeaking) {
+            // Speech started
+            this.isSpeaking = true;
+            this.speechStartTime = currentTime;
+            this.speechBuffer = Buffer.alloc(0);
+            console.log('Silero VAD: Speech started (prob:', speechProb.toFixed(3), ')');
+            // Emit the rising edge so the server can tell the client to barge
+            // in. Without this the client's 'speech_start' handler is dead and
+            // the assistant keeps talking over the user.
+            results.push({ type: 'speech_start', timestamp: currentTime });
+          }
+          
+          this.lastSpeechTime = currentTime;
+          this.speechBuffer = Buffer.concat([this.speechBuffer, chunk]);
+          
+        } else if (this.isSpeaking) {
+          // Check if silence duration exceeds threshold
+          const silenceDuration = currentTime - this.lastSpeechTime;
+          
+          if (silenceDuration > this.maxSilenceDuration) {
+            // Speech ended
+            const speechDuration = currentTime - this.speechStartTime;
+            
+            if (speechDuration >= this.minSpeechDuration) {
+              console.log('Silero VAD: Speech ended', { 
+                duration: speechDuration, 
+                bufferSize: this.speechBuffer.length 
+              });
+              
+              results.push({
+                type: 'speech_end',
+                audioData: this.speechBuffer,
+                duration: speechDuration,
+                observedSilenceDuration: silenceDuration,
+                endpointThreshold: this.maxSilenceDuration,
+                timestamp: currentTime
+              });
+            }
+            
+            this.isSpeaking = false;
+            this.speechStartTime = null;
+            this.lastSpeechTime = null;
+            this.speechBuffer = Buffer.alloc(0);
+          } else {
+            // Still in speech, add chunk to buffer
+            this.speechBuffer = Buffer.concat([this.speechBuffer, chunk]);
+          }
+        }
+        
+      } catch (error) {
+        console.error('Error processing audio chunk with Silero VAD:', error);
+        // Continue processing other chunks
+      }
+    }
+    
+    return results;
+  }
+
+  /**
+   * Force end current speech session
+   * @returns {Promise<Object|null>} Final speech data if available
+   */
+  async forceEndSpeech() {
+    if (this.isSpeaking && this.speechBuffer.length > 0) {
+      const currentTime = Date.now();
+      const speechDuration = currentTime - this.speechStartTime;
+      
+      if (speechDuration >= this.minSpeechDuration) {
+        const result = {
+          type: 'speech_end',
+          audioData: this.speechBuffer,
+          duration: speechDuration,
+          timestamp: currentTime
+        };
+        
+        this.isSpeaking = false;
+        this.speechStartTime = null;
+        this.lastSpeechTime = null;
+        this.speechBuffer = Buffer.alloc(0);
+        
+        return result;
+      }
+    }
+    
+    this.reset();
+    return null;
+  }
+
+  /**
+   * Reset VAD state
+   */
+  reset() {
+    this.isSpeaking = false;
+    this.speechStartTime = null;
+    this.lastSpeechTime = null;
+    this.audioBuffer = Buffer.alloc(0);
+    this.speechBuffer = Buffer.alloc(0);
+    
+    // Reset Silero VAD state
+    if (this.isInitialized) {
+      this.resetSileroState();
+    }
+  }
+
+  /**
+   * Get current VAD state
+   * @returns {Object} Current state information
+   */
+  getState() {
+    return {
+      isSpeaking: this.isSpeaking,
+      speechDuration: this.speechStartTime ? Date.now() - this.speechStartTime : 0,
+      bufferSize: this.speechBuffer.length,
+      threshold: this.threshold,
+      isInitialized: this.isInitialized
+    };
+  }
+
+  /**
+   * Cleanup resources
+   */
+  async cleanup() {
+    if (this.session) {
+      try {
+        await this.session.release();
+      } catch (error) {
+        console.error('Error releasing ONNX session:', error);
+      }
+      this.session = null;
+    }
+    
+    this.state = null;
+    this.sr = null;
+    this.isInitialized = false;
+    this.reset();
+  }
+}
+
+module.exports = VoiceActivityDetector;