mirror of https://github.com/TriliumNext/Notes
feat(llm): still working on fixing tests...
perf3ct
parent
c6f2124e9d
commit
d60e795421
@ -0,0 +1,538 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { processProviderStream, StreamProcessor } from '../providers/stream_handler.js';
|
||||
import type { ProviderStreamOptions } from '../providers/stream_handler.js';
|
||||
|
||||
// Mock log service
|
||||
vi.mock('../log.js', () => ({
|
||||
default: {
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
warn: vi.fn()
|
||||
}
|
||||
}));
|
||||
|
||||
describe('Streaming Error Handling Tests', () => {
|
||||
let mockOptions: ProviderStreamOptions;
|
||||
let log: any;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.clearAllMocks();
|
||||
log = (await import('../log.js')).default;
|
||||
mockOptions = {
|
||||
providerName: 'ErrorTestProvider',
|
||||
modelName: 'error-test-model'
|
||||
};
|
||||
});
|
||||
|
||||
describe('Stream Iterator Errors', () => {
|
||||
it('should handle iterator throwing error immediately', async () => {
|
||||
const errorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('Iterator initialization failed');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(errorIterator, mockOptions))
|
||||
.rejects.toThrow('Iterator initialization failed');
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error in ErrorTestProvider stream processing')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle iterator throwing error mid-stream', async () => {
|
||||
const midStreamErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Starting...' } };
|
||||
yield { message: { content: 'Processing...' } };
|
||||
throw new Error('Connection lost mid-stream');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(midStreamErrorIterator, mockOptions))
|
||||
.rejects.toThrow('Connection lost mid-stream');
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Connection lost mid-stream')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle async iterator returning invalid chunks', async () => {
|
||||
const invalidChunkIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield null; // Invalid chunk
|
||||
yield undefined; // Invalid chunk
|
||||
yield { randomField: 'not a valid chunk' };
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
// Should not throw, but handle gracefully
|
||||
const result = await processProviderStream(invalidChunkIterator, mockOptions);
|
||||
|
||||
expect(result.completeText).toBe('');
|
||||
expect(result.chunkCount).toBe(4);
|
||||
});
|
||||
|
||||
it('should handle iterator returning non-objects', async () => {
|
||||
const nonObjectIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield 'string chunk'; // Invalid
|
||||
yield 123; // Invalid
|
||||
yield true; // Invalid
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(nonObjectIterator, mockOptions);
|
||||
expect(result.completeText).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Callback Errors', () => {
|
||||
it('should handle callback throwing synchronous errors', async () => {
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Test' } };
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
const errorCallback = vi.fn(() => {
|
||||
throw new Error('Callback sync error');
|
||||
});
|
||||
|
||||
// Should not throw from main function
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
errorCallback
|
||||
);
|
||||
|
||||
expect(result.completeText).toBe('Test');
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error in streamCallback')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle callback throwing async errors', async () => {
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Test async' } };
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
const asyncErrorCallback = vi.fn(async () => {
|
||||
throw new Error('Callback async error');
|
||||
});
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
asyncErrorCallback
|
||||
);
|
||||
|
||||
expect(result.completeText).toBe('Test async');
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error in streamCallback')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle callback that never resolves', async () => {
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Hanging test' } };
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
const hangingCallback = vi.fn(async () => {
|
||||
// Never resolves
|
||||
return new Promise(() => {});
|
||||
});
|
||||
|
||||
// This test verifies we don't hang indefinitely
|
||||
const timeoutPromise = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('Test timeout')), 1000)
|
||||
);
|
||||
|
||||
const streamPromise = processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
hangingCallback
|
||||
);
|
||||
|
||||
// The stream should complete even if callback hangs
|
||||
// Note: This test design may need adjustment based on actual implementation
|
||||
await expect(Promise.race([streamPromise, timeoutPromise]))
|
||||
.rejects.toThrow('Test timeout');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Network and Connectivity Errors', () => {
|
||||
it('should handle network timeout errors', async () => {
|
||||
const timeoutIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Starting...' } };
|
||||
await new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('ECONNRESET: Connection reset by peer')), 100)
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(timeoutIterator, mockOptions))
|
||||
.rejects.toThrow('ECONNRESET');
|
||||
});
|
||||
|
||||
it('should handle DNS resolution errors', async () => {
|
||||
const dnsErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('ENOTFOUND: getaddrinfo ENOTFOUND api.invalid.domain');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(dnsErrorIterator, mockOptions))
|
||||
.rejects.toThrow('ENOTFOUND');
|
||||
});
|
||||
|
||||
it('should handle SSL/TLS certificate errors', async () => {
|
||||
const sslErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('UNABLE_TO_VERIFY_LEAF_SIGNATURE: certificate verify failed');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(sslErrorIterator, mockOptions))
|
||||
.rejects.toThrow('UNABLE_TO_VERIFY_LEAF_SIGNATURE');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provider-Specific Errors', () => {
|
||||
it('should handle OpenAI API errors', async () => {
|
||||
const openAIErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('Incorrect API key provided. Please check your API key.');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(
|
||||
openAIErrorIterator,
|
||||
{ ...mockOptions, providerName: 'OpenAI' }
|
||||
)).rejects.toThrow('Incorrect API key provided');
|
||||
});
|
||||
|
||||
it('should handle Anthropic rate limiting', async () => {
|
||||
const anthropicRateLimit = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Starting...' } };
|
||||
throw new Error('Rate limit exceeded. Please try again later.');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(
|
||||
anthropicRateLimit,
|
||||
{ ...mockOptions, providerName: 'Anthropic' }
|
||||
)).rejects.toThrow('Rate limit exceeded');
|
||||
});
|
||||
|
||||
it('should handle Ollama service unavailable', async () => {
|
||||
const ollamaUnavailable = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('Ollama service is not running. Please start Ollama first.');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(
|
||||
ollamaUnavailable,
|
||||
{ ...mockOptions, providerName: 'Ollama' }
|
||||
)).rejects.toThrow('Ollama service is not running');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Memory and Resource Errors', () => {
|
||||
it('should handle out of memory errors gracefully', async () => {
|
||||
const memoryErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Normal start' } };
|
||||
throw new Error('JavaScript heap out of memory');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(memoryErrorIterator, mockOptions))
|
||||
.rejects.toThrow('JavaScript heap out of memory');
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('JavaScript heap out of memory')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle file descriptor exhaustion', async () => {
|
||||
const fdErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('EMFILE: too many open files');
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(fdErrorIterator, mockOptions))
|
||||
.rejects.toThrow('EMFILE');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Streaming State Errors', () => {
|
||||
it('should handle chunks received after done=true', async () => {
|
||||
const postDoneIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Normal chunk' } };
|
||||
yield { message: { content: 'Final chunk' }, done: true };
|
||||
// These should be ignored or handled gracefully
|
||||
yield { message: { content: 'Post-done chunk 1' } };
|
||||
yield { message: { content: 'Post-done chunk 2' } };
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(postDoneIterator, mockOptions);
|
||||
|
||||
expect(result.completeText).toBe('Normal chunkFinal chunk');
|
||||
expect(result.chunkCount).toBe(4); // All chunks counted
|
||||
});
|
||||
|
||||
it('should handle multiple done=true chunks', async () => {
|
||||
const multipleDoneIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'Content' } };
|
||||
yield { done: true };
|
||||
yield { done: true }; // Duplicate done
|
||||
yield { done: true }; // Another duplicate
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(multipleDoneIterator, mockOptions);
|
||||
expect(result.chunkCount).toBe(4);
|
||||
});
|
||||
|
||||
it('should handle never-ending streams (no done flag)', async () => {
|
||||
let chunkCount = 0;
|
||||
const neverEndingIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
while (chunkCount < 1000) { // Simulate very long stream
|
||||
yield { message: { content: `chunk${chunkCount++}` } };
|
||||
if (chunkCount % 100 === 0) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
}
|
||||
}
|
||||
// Never yields done: true
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(neverEndingIterator, mockOptions);
|
||||
|
||||
expect(result.chunkCount).toBe(1000);
|
||||
expect(result.completeText).toContain('chunk999');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Concurrent Error Scenarios', () => {
|
||||
it('should handle errors during concurrent streaming', async () => {
|
||||
const createFailingIterator = (failAt: number) => ({
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
if (i === failAt) {
|
||||
throw new Error(`Concurrent error at chunk ${i}`);
|
||||
}
|
||||
yield { message: { content: `chunk${i}` } };
|
||||
}
|
||||
yield { done: true };
|
||||
}
|
||||
});
|
||||
|
||||
// Start multiple streams, some will fail
|
||||
const promises = [
|
||||
processProviderStream(createFailingIterator(3), mockOptions),
|
||||
processProviderStream(createFailingIterator(5), mockOptions),
|
||||
processProviderStream(createFailingIterator(7), mockOptions)
|
||||
];
|
||||
|
||||
const results = await Promise.allSettled(promises);
|
||||
|
||||
// All should be rejected
|
||||
results.forEach(result => {
|
||||
expect(result.status).toBe('rejected');
|
||||
if (result.status === 'rejected') {
|
||||
expect(result.reason.message).toMatch(/Concurrent error at chunk \d/);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it('should isolate errors between concurrent streams', async () => {
|
||||
const goodIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
yield { message: { content: `good${i}` } };
|
||||
await new Promise(resolve => setTimeout(resolve, 10));
|
||||
}
|
||||
yield { done: true };
|
||||
}
|
||||
};
|
||||
|
||||
const badIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
yield { message: { content: 'bad start' } };
|
||||
throw new Error('Bad stream error');
|
||||
}
|
||||
};
|
||||
|
||||
const [goodResult, badResult] = await Promise.allSettled([
|
||||
processProviderStream(goodIterator, mockOptions),
|
||||
processProviderStream(badIterator, mockOptions)
|
||||
]);
|
||||
|
||||
expect(goodResult.status).toBe('fulfilled');
|
||||
expect(badResult.status).toBe('rejected');
|
||||
|
||||
if (goodResult.status === 'fulfilled') {
|
||||
expect(goodResult.value.completeText).toContain('good4');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Recovery and Cleanup', () => {
|
||||
it('should clean up resources on error', async () => {
|
||||
let resourcesAllocated = false;
|
||||
let resourcesCleaned = false;
|
||||
|
||||
const resourceErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
resourcesAllocated = true;
|
||||
try {
|
||||
yield { message: { content: 'Resource test' } };
|
||||
throw new Error('Resource allocation failed');
|
||||
} finally {
|
||||
resourcesCleaned = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(resourceErrorIterator, mockOptions))
|
||||
.rejects.toThrow('Resource allocation failed');
|
||||
|
||||
expect(resourcesAllocated).toBe(true);
|
||||
expect(resourcesCleaned).toBe(true);
|
||||
});
|
||||
|
||||
it('should log comprehensive error details', async () => {
|
||||
const detailedError = new Error('Detailed test error');
|
||||
detailedError.stack = 'Error: Detailed test error\n at test location\n at another location';
|
||||
|
||||
const errorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw detailedError;
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(errorIterator, mockOptions))
|
||||
.rejects.toThrow('Detailed test error');
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error in ErrorTestProvider stream processing: Detailed test error')
|
||||
);
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error details:')
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle errors in error logging', async () => {
|
||||
// Mock log.error to throw
|
||||
log.error.mockImplementation(() => {
|
||||
throw new Error('Logging failed');
|
||||
});
|
||||
|
||||
const errorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw new Error('Original error');
|
||||
}
|
||||
};
|
||||
|
||||
// Should still propagate original error, not logging error
|
||||
await expect(processProviderStream(errorIterator, mockOptions))
|
||||
.rejects.toThrow('Original error');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Edge Case Error Scenarios', () => {
|
||||
it('should handle errors with circular references', async () => {
|
||||
const circularError: any = new Error('Circular error');
|
||||
circularError.circular = circularError;
|
||||
|
||||
const circularErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw circularError;
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(circularErrorIterator, mockOptions))
|
||||
.rejects.toThrow('Circular error');
|
||||
});
|
||||
|
||||
it('should handle non-Error objects being thrown', async () => {
|
||||
const nonErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw 'String error'; // Not an Error object
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(nonErrorIterator, mockOptions))
|
||||
.rejects.toBe('String error');
|
||||
});
|
||||
|
||||
it('should handle undefined/null being thrown', async () => {
|
||||
const nullErrorIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
throw null;
|
||||
}
|
||||
};
|
||||
|
||||
await expect(processProviderStream(nullErrorIterator, mockOptions))
|
||||
.rejects.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('StreamProcessor Error Handling', () => {
|
||||
it('should handle malformed chunk processing', async () => {
|
||||
const malformedChunk = {
|
||||
message: {
|
||||
content: { not: 'a string' } // Should be string
|
||||
}
|
||||
};
|
||||
|
||||
const result = await StreamProcessor.processChunk(
|
||||
malformedChunk,
|
||||
'',
|
||||
1,
|
||||
{ providerName: 'Test', modelName: 'test' }
|
||||
);
|
||||
|
||||
// Should handle gracefully without throwing
|
||||
expect(result.completeText).toBe('');
|
||||
});
|
||||
|
||||
it('should handle callback errors in sendChunkToCallback', async () => {
|
||||
const errorCallback = vi.fn(() => {
|
||||
throw new Error('Callback processing error');
|
||||
});
|
||||
|
||||
// Should not throw
|
||||
await expect(StreamProcessor.sendChunkToCallback(
|
||||
errorCallback,
|
||||
'test content',
|
||||
false,
|
||||
{},
|
||||
1
|
||||
)).resolves.toBeUndefined();
|
||||
|
||||
expect(log.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining('Error in streamCallback')
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,678 @@
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { processProviderStream, StreamProcessor } from '../providers/stream_handler.js';
|
||||
import type { ProviderStreamOptions } from '../providers/stream_handler.js';
|
||||
|
||||
// Mock log service
|
||||
vi.mock('../log.js', () => ({
|
||||
default: {
|
||||
info: vi.fn(),
|
||||
error: vi.fn(),
|
||||
warn: vi.fn()
|
||||
}
|
||||
}));
|
||||
|
||||
describe('Tool Execution During Streaming Tests', () => {
|
||||
let mockOptions: ProviderStreamOptions;
|
||||
let receivedCallbacks: Array<{ text: string; done: boolean; chunk: any }>;
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
receivedCallbacks = [];
|
||||
mockOptions = {
|
||||
providerName: 'ToolTestProvider',
|
||||
modelName: 'tool-capable-model'
|
||||
};
|
||||
});
|
||||
|
||||
const mockCallback = (text: string, done: boolean, chunk: any) => {
|
||||
receivedCallbacks.push({ text, done, chunk });
|
||||
};
|
||||
|
||||
describe('Basic Tool Call Handling', () => {
|
||||
it('should extract and process simple tool calls', async () => {
|
||||
const toolChunks = [
|
||||
{ message: { content: 'Let me search for that' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_search_123',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'web_search',
|
||||
arguments: '{"query": "weather today"}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'The weather today is sunny.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of toolChunks) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls[0]).toEqual({
|
||||
id: 'call_search_123',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'web_search',
|
||||
arguments: '{"query": "weather today"}'
|
||||
}
|
||||
});
|
||||
expect(result.completeText).toBe('Let me search for thatThe weather today is sunny.');
|
||||
});
|
||||
|
||||
it('should handle multiple tool calls in sequence', async () => {
|
||||
const multiToolChunks = [
|
||||
{ message: { content: 'I need to use multiple tools' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_1',
|
||||
function: { name: 'calculator', arguments: '{"expr": "2+2"}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'First calculation complete. Now searching...' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_2',
|
||||
function: { name: 'web_search', arguments: '{"query": "math"}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'All tasks completed.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of multiToolChunks) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
// Should capture the last tool calls (overwriting previous ones as per implementation)
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
expect(result.toolCalls[0].function.name).toBe('web_search');
|
||||
});
|
||||
|
||||
it('should handle tool calls with complex arguments', async () => {
|
||||
const complexToolChunk = {
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_complex',
|
||||
function: {
|
||||
name: 'data_processor',
|
||||
arguments: JSON.stringify({
|
||||
dataset: {
|
||||
source: 'database',
|
||||
filters: { active: true, category: 'sales' },
|
||||
columns: ['id', 'name', 'amount', 'date']
|
||||
},
|
||||
operations: [
|
||||
{ type: 'filter', condition: 'amount > 100' },
|
||||
{ type: 'group', by: 'category' },
|
||||
{ type: 'aggregate', function: 'sum', column: 'amount' }
|
||||
],
|
||||
output: { format: 'json', include_metadata: true }
|
||||
})
|
||||
}
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(complexToolChunk);
|
||||
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
expect(toolCalls[0].function.name).toBe('data_processor');
|
||||
|
||||
const args = JSON.parse(toolCalls[0].function.arguments);
|
||||
expect(args.dataset.source).toBe('database');
|
||||
expect(args.operations).toHaveLength(3);
|
||||
expect(args.output.format).toBe('json');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Call Extraction Edge Cases', () => {
|
||||
it('should handle empty tool_calls array', async () => {
|
||||
const emptyToolChunk = {
|
||||
message: {
|
||||
content: 'No tools needed',
|
||||
tool_calls: []
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(emptyToolChunk);
|
||||
expect(toolCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle malformed tool_calls', async () => {
|
||||
const malformedChunk = {
|
||||
message: {
|
||||
tool_calls: 'not an array'
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(malformedChunk);
|
||||
expect(toolCalls).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle missing function field in tool call', async () => {
|
||||
const incompleteToolChunk = {
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_incomplete',
|
||||
type: 'function'
|
||||
// Missing function field
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(incompleteToolChunk);
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
expect(toolCalls[0].id).toBe('call_incomplete');
|
||||
});
|
||||
|
||||
it('should handle tool calls with invalid JSON arguments', async () => {
|
||||
const invalidJsonChunk = {
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_invalid_json',
|
||||
function: {
|
||||
name: 'test_tool',
|
||||
arguments: '{"invalid": json}' // Invalid JSON
|
||||
}
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(invalidJsonChunk);
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
expect(toolCalls[0].function.arguments).toBe('{"invalid": json}');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Real-world Tool Execution Scenarios', () => {
|
||||
it('should handle calculator tool execution', async () => {
|
||||
const calculatorScenario = [
|
||||
{ message: { content: 'Let me calculate that for you' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_calc_456',
|
||||
function: {
|
||||
name: 'calculator',
|
||||
arguments: '{"expression": "15 * 37 + 22"}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'The result is 577.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of calculatorScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.toolCalls[0].function.name).toBe('calculator');
|
||||
expect(result.completeText).toBe('Let me calculate that for youThe result is 577.');
|
||||
});
|
||||
|
||||
it('should handle web search tool execution', async () => {
|
||||
const searchScenario = [
|
||||
{ message: { content: 'Searching for current information...' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_search_789',
|
||||
function: {
|
||||
name: 'web_search',
|
||||
arguments: '{"query": "latest AI developments 2024", "num_results": 5}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Based on my search, here are the latest AI developments...' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of searchScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.toolCalls[0].function.name).toBe('web_search');
|
||||
const args = JSON.parse(result.toolCalls[0].function.arguments);
|
||||
expect(args.num_results).toBe(5);
|
||||
});
|
||||
|
||||
it('should handle file operations tool execution', async () => {
|
||||
const fileOpScenario = [
|
||||
{ message: { content: 'I\'ll help you analyze that file' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_file_read',
|
||||
function: {
|
||||
name: 'read_file',
|
||||
arguments: '{"path": "/data/report.csv", "encoding": "utf-8"}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'File contents analyzed. The report contains...' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_file_write',
|
||||
function: {
|
||||
name: 'write_file',
|
||||
arguments: '{"path": "/data/summary.txt", "content": "Analysis summary..."}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Summary saved successfully.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of fileOpScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
// Should have the last tool call
|
||||
expect(result.toolCalls[0].function.name).toBe('write_file');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Execution with Content Streaming', () => {
|
||||
it('should interleave tool calls with content correctly', async () => {
|
||||
const interleavedScenario = [
|
||||
{ message: { content: 'Starting analysis' } },
|
||||
{
|
||||
message: {
|
||||
content: ' with tools.',
|
||||
tool_calls: [{
|
||||
id: 'call_analyze',
|
||||
function: { name: 'analyzer', arguments: '{}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: ' Tool executed.' } },
|
||||
{ message: { content: ' Final results ready.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of interleavedScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.completeText).toBe('Starting analysis with tools. Tool executed. Final results ready.');
|
||||
expect(result.toolCalls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should handle tool calls without content in same chunk', async () => {
|
||||
const toolOnlyChunks = [
|
||||
{ message: { content: 'Preparing to use tools' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_tool_only',
|
||||
function: { name: 'silent_tool', arguments: '{}' }
|
||||
}]
|
||||
// No content in this chunk
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Tool completed silently' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of toolOnlyChunks) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.completeText).toBe('Preparing to use toolsTool completed silently');
|
||||
expect(result.toolCalls[0].function.name).toBe('silent_tool');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Provider-Specific Tool Formats', () => {
|
||||
it('should handle OpenAI tool call format', async () => {
|
||||
const openAIToolFormat = {
|
||||
choices: [{
|
||||
delta: {
|
||||
tool_calls: [{
|
||||
index: 0,
|
||||
id: 'call_openai_123',
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_weather',
|
||||
arguments: '{"location": "San Francisco"}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
}]
|
||||
};
|
||||
|
||||
// Convert to our standard format for testing
|
||||
const standardFormat = {
|
||||
message: {
|
||||
tool_calls: openAIToolFormat.choices[0].delta.tool_calls
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(standardFormat);
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
expect(toolCalls[0].function.name).toBe('get_weather');
|
||||
});
|
||||
|
||||
it('should handle Anthropic tool call format', async () => {
|
||||
// Anthropic uses different format - simulate conversion
|
||||
const anthropicToolData = {
|
||||
type: 'tool_use',
|
||||
id: 'call_anthropic_456',
|
||||
name: 'search_engine',
|
||||
input: { query: 'best restaurants nearby' }
|
||||
};
|
||||
|
||||
// Convert to our standard format
|
||||
const standardFormat = {
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: anthropicToolData.id,
|
||||
function: {
|
||||
name: anthropicToolData.name,
|
||||
arguments: JSON.stringify(anthropicToolData.input)
|
||||
}
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(standardFormat);
|
||||
expect(toolCalls).toHaveLength(1);
|
||||
expect(toolCalls[0].function.name).toBe('search_engine');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Execution Error Scenarios', () => {
|
||||
it('should handle tool execution errors in stream', async () => {
|
||||
const toolErrorScenario = [
|
||||
{ message: { content: 'Attempting tool execution' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_error_test',
|
||||
function: {
|
||||
name: 'failing_tool',
|
||||
arguments: '{"param": "value"}'
|
||||
}
|
||||
}]
|
||||
}
|
||||
},
|
||||
{
|
||||
message: {
|
||||
content: 'Tool execution failed: Permission denied',
|
||||
error: 'Tool execution error'
|
||||
}
|
||||
},
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of toolErrorScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.toolCalls[0].function.name).toBe('failing_tool');
|
||||
expect(result.completeText).toContain('Tool execution failed');
|
||||
});
|
||||
|
||||
it('should handle timeout in tool execution', async () => {
|
||||
const timeoutScenario = [
|
||||
{ message: { content: 'Starting long-running tool' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'call_timeout',
|
||||
function: { name: 'slow_tool', arguments: '{}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Tool timed out after 30 seconds' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of timeoutScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
expect(result.completeText).toContain('timed out');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Complex Tool Workflows', () => {
|
||||
it('should handle multi-step tool workflow', async () => {
|
||||
const workflowScenario = [
|
||||
{ message: { content: 'Starting multi-step analysis' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'step1',
|
||||
function: { name: 'data_fetch', arguments: '{"source": "api"}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Data fetched. Processing...' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'step2',
|
||||
function: { name: 'data_process', arguments: '{"format": "json"}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Processing complete. Generating report...' } },
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'step3',
|
||||
function: { name: 'report_generate', arguments: '{"type": "summary"}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ message: { content: 'Workflow completed successfully.' } },
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of workflowScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const result = await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
// Should capture the last tool call
|
||||
expect(result.toolCalls[0].function.name).toBe('report_generate');
|
||||
expect(result.completeText).toContain('Workflow completed successfully');
|
||||
});
|
||||
|
||||
it('should handle parallel tool execution indication', async () => {
|
||||
const parallelToolsChunk = {
|
||||
message: {
|
||||
tool_calls: [
|
||||
{
|
||||
id: 'parallel_1',
|
||||
function: { name: 'fetch_weather', arguments: '{"city": "NYC"}' }
|
||||
},
|
||||
{
|
||||
id: 'parallel_2',
|
||||
function: { name: 'fetch_news', arguments: '{"topic": "technology"}' }
|
||||
},
|
||||
{
|
||||
id: 'parallel_3',
|
||||
function: { name: 'fetch_stocks', arguments: '{"symbol": "AAPL"}' }
|
||||
}
|
||||
]
|
||||
}
|
||||
};
|
||||
|
||||
const toolCalls = StreamProcessor.extractToolCalls(parallelToolsChunk);
|
||||
expect(toolCalls).toHaveLength(3);
|
||||
expect(toolCalls.map(tc => tc.function.name)).toEqual([
|
||||
'fetch_weather', 'fetch_news', 'fetch_stocks'
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Tool Call Logging and Debugging', () => {
|
||||
it('should log tool call detection', async () => {
|
||||
const log = (await import('../log.js')).default;
|
||||
|
||||
const toolChunk = {
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'log_test',
|
||||
function: { name: 'test_tool', arguments: '{}' }
|
||||
}]
|
||||
}
|
||||
};
|
||||
|
||||
StreamProcessor.extractToolCalls(toolChunk);
|
||||
|
||||
expect(log.info).toHaveBeenCalledWith(
|
||||
'Detected 1 tool calls in stream chunk'
|
||||
);
|
||||
});
|
||||
|
||||
it('should handle tool calls in callback correctly', async () => {
|
||||
const toolCallbackScenario = [
|
||||
{
|
||||
message: {
|
||||
tool_calls: [{
|
||||
id: 'callback_test',
|
||||
function: { name: 'callback_tool', arguments: '{"test": true}' }
|
||||
}]
|
||||
}
|
||||
},
|
||||
{ done: true }
|
||||
];
|
||||
|
||||
const mockIterator = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
for (const chunk of toolCallbackScenario) {
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await processProviderStream(
|
||||
mockIterator,
|
||||
mockOptions,
|
||||
mockCallback
|
||||
);
|
||||
|
||||
// Should have received callback for tool execution chunk
|
||||
const toolCallbacks = receivedCallbacks.filter(cb =>
|
||||
cb.chunk && cb.chunk.message && cb.chunk.message.tool_calls
|
||||
);
|
||||
expect(toolCallbacks.length).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue