const assert = require('assert'); // Import LLM provider factory const { LLMProviderFactory } = require('../utils/providers/llmProviders'); // Test configuration const testConfig = { OPENAI_API_KEY: process.env.OPENAI_API_KEY, OPENROUTER_API_KEY: process.env.OPENROUTER_API_KEY, ARK_API_KEY: process.env.ARK_API_KEY, SILICONFLOW_API_KEY: process.env.SILICONFLOW_API_KEY, LLM_PROVIDERS: { openai: { apiUrl: 'https://api.openai.com/v1/chat/completions', model: 'gpt-5.6-luna', apiKey: 'OPENAI_API_KEY' }, 'openrouter-gpt': { apiUrl: 'https://openrouter.ai/api/v1/chat/completions', model: 'openai/gpt-5.6-luna', apiKey: 'OPENROUTER_API_KEY' }, 'openrouter-gemini': { apiUrl: 'https://openrouter.ai/api/v1/chat/completions', model: 'google/gemini-3.5-flash', apiKey: 'OPENROUTER_API_KEY' }, ark: { apiUrl: 'https://ark.cn-beijing.volces.com/api/v3/chat/completions', model: 'doubao-seed-1-6-flash-250615', apiKey: 'ARK_API_KEY' } }, VISION_MAX_TOKENS: 4096 }; // Test messages for different scenarios const testMessages = { simple: [ { role: 'system', content: 'You are a helpful AI assistant.' }, { role: 'user', content: 'Hello! Please respond with exactly the word "SUCCESS" to confirm you are working.' } ], conversation: [ { role: 'system', content: 'You are a conversational AI assistant.' }, { role: 'user', content: 'What is the capital of France?' }, { role: 'assistant', content: 'The capital of France is Paris.' }, { role: 'user', content: 'What is its population?' } ], creative: [ { role: 'system', content: 'You are a creative writing assistant.' }, { role: 'user', content: 'Write a very short story about a robot learning to paint. Keep it under 50 words.' } ] }; /** * Collect streaming response data */ async function collectStreamingResponse(response, timeout = 30000) { return new Promise((resolve, reject) => { let buffer = ''; let accumulatedContent = ''; let firstTokenTime = null; let tokenCount = 0; const timeoutHandle = setTimeout(() => { reject(new Error('Streaming response timeout')); }, timeout); response.data.on('data', (chunk) => { try { if (!firstTokenTime) { firstTokenTime = Date.now(); } buffer += chunk.toString(); const lines = buffer.split('\n'); // Keep the last line if it's incomplete buffer = lines.pop() || ''; for (const line of lines) { const trimmedLine = line.trim(); if (!trimmedLine || trimmedLine === '[DONE]') continue; if (!trimmedLine.startsWith('data: ')) continue; try { const jsonData = JSON.parse(trimmedLine.replace('data: ', '')); const content = jsonData.choices[0]?.delta?.content || ''; if (content) { accumulatedContent += content; tokenCount++; } } catch (parseError) { // Skip malformed JSON continue; } } } catch (error) { reject(error); } }); response.data.on('end', () => { clearTimeout(timeoutHandle); resolve({ content: accumulatedContent, tokenCount: tokenCount, firstTokenTime: firstTokenTime, success: true }); }); response.data.on('error', (error) => { clearTimeout(timeoutHandle); reject(error); }); }); } /** * LLM Provider Individual Tests */ describe('LLM Providers - Individual Testing', function() { this.timeout(120000); // 2 minute timeout for API calls describe('OpenAI GPT-4o LLM Provider', function() { let provider; before(function() { if (!testConfig.OPENAI_API_KEY) { console.log('⚠️ Skipping OpenAI LLM tests - OPENAI_API_KEY not found'); this.skip(); } try { provider = LLMProviderFactory.createProvider('openai', testConfig, testConfig); console.log('βœ… OpenAI LLM Provider created successfully'); } catch (error) { console.error('❌ Failed to create OpenAI LLM provider:', error); this.skip(); } }); it('should initialize with correct configuration', function() { assert(provider, 'Provider should be created'); assert.strictEqual(provider.config.model, 'gpt-5.6-luna', 'Should use gpt-5.6-luna model'); assert.strictEqual(provider.config.apiUrl, 'https://api.openai.com/v1/chat/completions', 'Should use correct API URL'); assert(provider.apiKey, 'Should have API key'); console.log('πŸ“‹ OpenAI LLM Config:', { model: provider.config.model, apiUrl: provider.config.apiUrl }); }); it('should generate simple chat completion', async function() { console.log('πŸ€– Testing OpenAI LLM with simple message'); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 100 }); const responseTime = Date.now() - startTime; console.log('πŸ“Š OpenAI LLM Simple Test:', { success: result.success, responseTime: `${responseTime}ms`, provider: result.provider }); assert(result, 'Should return a result'); assert(typeof result.success === 'boolean', 'Should have success field'); assert.strictEqual(result.provider, 'openai', 'Should identify as openai provider'); if (result.success) { assert(result.response, 'Should have response object'); console.log('βœ… OpenAI LLM simple completion successful'); } else { console.log('❌ OpenAI LLM simple completion failed:', result.error); } }); it('should handle streaming chat completion', async function() { console.log('🌊 Testing OpenAI LLM streaming'); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 50, stream: true }); if (!result.success) { console.log('❌ OpenAI LLM streaming setup failed:', result.error); assert(false, 'Streaming setup should succeed'); return; } try { const streamResult = await collectStreamingResponse(result.response); const totalTime = Date.now() - startTime; const timeToFirstToken = streamResult.firstTokenTime ? streamResult.firstTokenTime - startTime : 0; console.log('πŸ“Š OpenAI LLM Streaming Results:', { success: streamResult.success, totalTime: `${totalTime}ms`, timeToFirstToken: `${timeToFirstToken}ms`, tokenCount: streamResult.tokenCount, contentLength: streamResult.content.length }); assert(streamResult.success, 'Streaming should be successful'); assert(streamResult.content.length > 0, 'Should receive content'); assert(streamResult.tokenCount > 0, 'Should receive tokens'); if (streamResult.content.length > 0) { console.log('πŸ“ Generated content preview:', streamResult.content.substring(0, 100) + (streamResult.content.length > 100 ? '...' : '')); } console.log('βœ… OpenAI LLM streaming successful'); } catch (streamError) { console.error('❌ OpenAI LLM streaming failed:', streamError.message); assert(false, 'Streaming should not fail: ' + streamError.message); } }); it('should handle conversation context', async function() { console.log('πŸ’¬ Testing OpenAI LLM with conversation context'); const result = await provider.createChatCompletion(testMessages.conversation, { max_tokens: 100 }); console.log('πŸ“Š OpenAI LLM Conversation Test:', { success: result.success, provider: result.provider }); assert(result, 'Should return a result'); assert(typeof result.success === 'boolean', 'Should have success field'); if (result.success) { console.log('βœ… OpenAI LLM conversation handling successful'); } else { console.log('❌ OpenAI LLM conversation handling failed:', result.error); } }); }); describe('OpenRouter GPT LLM Provider', function() { let provider; before(function() { if (!testConfig.OPENROUTER_API_KEY) { console.log('⚠️ Skipping OpenRouter GPT tests - OPENROUTER_API_KEY not found'); this.skip(); } try { provider = LLMProviderFactory.createProvider('openrouter-gpt', testConfig, testConfig); console.log('βœ… OpenRouter GPT Provider created successfully'); } catch (error) { console.error('❌ Failed to create OpenRouter GPT provider:', error); this.skip(); } }); it('should initialize with correct configuration', function() { assert(provider, 'Provider should be created'); assert.strictEqual(provider.config.model, 'openai/gpt-5.6-luna', 'Should use openai/gpt-5.6-luna model'); assert.strictEqual(provider.config.apiUrl, 'https://openrouter.ai/api/v1/chat/completions', 'Should use OpenRouter API URL'); assert(provider.apiKey, 'Should have API key'); console.log('πŸ“‹ OpenRouter GPT Config:', { model: provider.config.model, apiUrl: provider.config.apiUrl }); }); it('should generate chat completion via OpenRouter', async function() { console.log('πŸ€– Testing OpenRouter GPT'); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 100 }); const responseTime = Date.now() - startTime; console.log('πŸ“Š OpenRouter GPT Test:', { success: result.success, responseTime: `${responseTime}ms`, provider: result.provider }); assert(result, 'Should return a result'); assert(typeof result.success === 'boolean', 'Should have success field'); assert.strictEqual(result.provider, 'openrouter', 'Should identify as openrouter provider'); if (result.success) { console.log('βœ… OpenRouter GPT completion successful'); } else { console.log('❌ OpenRouter GPT completion failed:', result.error); } }); it('should handle streaming via OpenRouter', async function() { console.log('🌊 Testing OpenRouter GPT streaming'); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 50, stream: true }); if (!result.success) { console.log('❌ OpenRouter GPT streaming setup failed:', result.error); return; // Don't fail the test, just log } try { const streamResult = await collectStreamingResponse(result.response); console.log('πŸ“Š OpenRouter GPT Streaming:', { success: streamResult.success, tokenCount: streamResult.tokenCount, contentLength: streamResult.content.length }); if (streamResult.success) { console.log('βœ… OpenRouter GPT streaming successful'); } } catch (streamError) { console.log('❌ OpenRouter GPT streaming error:', streamError.message); } }); }); describe('OpenRouter Gemini LLM Provider', function() { let provider; before(function() { if (!testConfig.OPENROUTER_API_KEY) { console.log('⚠️ Skipping OpenRouter Gemini tests - OPENROUTER_API_KEY not found'); this.skip(); } try { provider = LLMProviderFactory.createProvider('openrouter-gemini', testConfig, testConfig); console.log('βœ… OpenRouter Gemini Provider created successfully'); } catch (error) { console.error('❌ Failed to create OpenRouter Gemini provider:', error); this.skip(); } }); it('should initialize with correct configuration', function() { assert(provider, 'Provider should be created'); assert.strictEqual(provider.config.model, 'google/gemini-3.5-flash', 'Should use gemini-3.5-flash model'); assert.strictEqual(provider.config.apiUrl, 'https://openrouter.ai/api/v1/chat/completions', 'Should use OpenRouter API URL'); assert(provider.apiKey, 'Should have API key'); console.log('πŸ“‹ OpenRouter Gemini Config:', { model: provider.config.model, apiUrl: provider.config.apiUrl }); }); it('should generate chat completion with Gemini', async function() { console.log('πŸ€– Testing OpenRouter Gemini'); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 100 }); const responseTime = Date.now() - startTime; console.log('πŸ“Š OpenRouter Gemini Test:', { success: result.success, responseTime: `${responseTime}ms`, provider: result.provider }); assert(result, 'Should return a result'); assert(typeof result.success === 'boolean', 'Should have success field'); assert.strictEqual(result.provider, 'openrouter', 'Should identify as openrouter provider'); if (result.success) { console.log('βœ… OpenRouter Gemini completion successful'); } else { console.log('❌ OpenRouter Gemini completion failed:', result.error); } }); it('should handle creative tasks with Gemini', async function() { console.log('🎨 Testing OpenRouter Gemini creative task'); const result = await provider.createChatCompletion(testMessages.creative, { max_tokens: 150 }); console.log('πŸ“Š OpenRouter Gemini Creative Test:', { success: result.success, provider: result.provider }); if (result.success) { console.log('βœ… OpenRouter Gemini creative task successful'); } else { console.log('❌ OpenRouter Gemini creative task failed:', result.error); } }); }); describe('ARK Doubao LLM Provider', function() { let provider; before(function() { if (!testConfig.ARK_API_KEY) { console.log('⚠️ Skipping ARK Doubao tests - ARK_API_KEY not found'); this.skip(); } try { provider = LLMProviderFactory.createProvider('ark', testConfig, testConfig); console.log('βœ… ARK Doubao Provider created successfully'); } catch (error) { console.error('❌ Failed to create ARK Doubao provider:', error); this.skip(); } }); it('should initialize with correct configuration', function() { assert(provider, 'Provider should be created'); assert.strictEqual(provider.config.model, 'doubao-seed-1-6-flash-250615', 'Should use doubao model'); assert.strictEqual(provider.config.apiUrl, 'https://ark.cn-beijing.volces.com/api/v3/chat/completions', 'Should use ARK API URL'); assert(provider.apiKey, 'Should have API key'); console.log('πŸ“‹ ARK Doubao Config:', { model: provider.config.model, apiUrl: provider.config.apiUrl }); }); it('should generate chat completion with Doubao', async function() { console.log('πŸ€– Testing ARK Doubao'); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 100 }); const responseTime = Date.now() - startTime; console.log('πŸ“Š ARK Doubao Test:', { success: result.success, responseTime: `${responseTime}ms`, provider: result.provider }); assert(result, 'Should return a result'); assert(typeof result.success === 'boolean', 'Should have success field'); assert.strictEqual(result.provider, 'ark', 'Should identify as ark provider'); if (result.success) { console.log('βœ… ARK Doubao completion successful'); } else { console.log('❌ ARK Doubao completion failed:', result.error); } }); it('should handle Chinese language tasks', async function() { console.log('πŸ‡¨πŸ‡³ Testing ARK Doubao with Chinese'); const chineseMessages = [ { role: 'system', content: 'δ½ ζ˜―δΈ€δΈͺζœ‰η”¨ηš„AIεŠ©ζ‰‹γ€‚' }, { role: 'user', content: '请用一ε₯θ―δ»‹η»εŒ—δΊ¬γ€‚' } ]; const result = await provider.createChatCompletion(chineseMessages, { max_tokens: 100 }); console.log('πŸ“Š ARK Doubao Chinese Test:', { success: result.success, provider: result.provider }); if (result.success) { console.log('βœ… ARK Doubao Chinese handling successful'); } else { console.log('❌ ARK Doubao Chinese handling failed:', result.error); } }); }); describe('LLM Provider Performance Comparison', function() { it('should compare provider response times', async function() { const availableProviders = []; if (testConfig.OPENAI_API_KEY) availableProviders.push('openai'); if (testConfig.OPENROUTER_API_KEY) { availableProviders.push('openrouter-gpt', 'openrouter-gemini'); } if (testConfig.ARK_API_KEY) availableProviders.push('ark'); if (availableProviders.length < 2) { console.log('⚠️ Skipping provider comparison - need at least 2 providers'); this.skip(); } const results = {}; console.log('🏁 Comparing LLM provider performance...'); for (const providerName of availableProviders) { try { const provider = LLMProviderFactory.createProvider(providerName, testConfig, testConfig); const startTime = Date.now(); const result = await provider.createChatCompletion(testMessages.simple, { max_tokens: 50 }); const duration = Date.now() - startTime; results[providerName] = { success: result.success, duration: duration, error: result.error }; } catch (error) { results[providerName] = { success: false, duration: 0, error: error.message }; } } console.log('πŸ“Š LLM Provider Performance Comparison:'); Object.entries(results).forEach(([provider, result]) => { console.log(` ${provider}:`, { success: result.success ? 'βœ…' : '❌', time: `${result.duration}ms`, error: result.error || 'none' }); }); // Find fastest successful provider const successfulProviders = Object.entries(results).filter(([_, result]) => result.success); if (successfulProviders.length > 0) { const fastest = successfulProviders.reduce((prev, curr) => prev[1].duration < curr[1].duration ? prev : curr ); console.log(`πŸ† Fastest LLM provider: ${fastest[0]} (${fastest[1].duration}ms)`); } // At least one provider should work assert(successfulProviders.length > 0, 'At least one LLM provider should be successful'); }); }); }); module.exports = { testConfig, testMessages, collectStreamingResponse };