444060c3eb
- Introduced `critic`, an independent adversarial reviewer for security and correctness. - Added `fable-orchestrator` to manage task routing and verification. - Implemented `gauntlet-critic` for fresh-context evaluation of gauntlet rounds. - Created `planner` for generating executable implementation plans with dependencies. - Developed `security-auditor` for application security reviews and audits. - Established `system-steward` to improve agent prompts and skills based on verified failures. - Added `dev-loop` skill for autonomous development loops over repositories. - Implemented `gauntlet-loop` skill for iterative quality benchmarking against reference standards. - Updated project settings to utilize the new orchestrator agent. - Created documentation for `GAUNTLET.md`, `PROGRESS.md`, and `REFERENCE_BAR.md` to track project status and quality benchmarks. - Added detailed prompting style guide to enhance understanding of prompt patterns and agentic loops.
383 lines
16 KiB
TypeScript
383 lines
16 KiB
TypeScript
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
import {
|
|
callProvider,
|
|
listModels,
|
|
getSystemPrompt,
|
|
defaultMaxTokens,
|
|
providerLabel,
|
|
PROVIDER_SPECS,
|
|
} from '@lib/providers';
|
|
import { PROMPT_PATTERNS, resolvePromptPattern } from '@lib/actions';
|
|
|
|
function mockFetchOnce(data: unknown, { ok = true, status = 200 } = {}) {
|
|
const fn = vi.fn().mockResolvedValue({ ok, status, json: async () => data });
|
|
vi.stubGlobal('fetch', fn);
|
|
return fn;
|
|
}
|
|
|
|
beforeEach(() => vi.restoreAllMocks());
|
|
afterEach(() => vi.unstubAllGlobals());
|
|
|
|
// ─── Request shapes pinned to the ORIGINAL callX/callXWithPrompt functions ────
|
|
// These bodies/headers/URLs are copied from the pre-refactor background.ts.
|
|
// maxTokens is forced to 1024 to match the old hard-coded value exactly.
|
|
|
|
const openAiResponse = { choices: [{ message: { content: ' fixed text ' } }] };
|
|
const anthropicResponse = { content: [{ text: ' fixed text ' }] };
|
|
|
|
describe('callProvider request shapes (parity with old implementations)', () => {
|
|
it('OpenAI: url, bearer auth, system message, temperature 0.7, default model', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
const res = await callProvider({ provider: 'openai', apiKey: 'sk-x' }, 'hello', 'SYS', { maxTokens: 1024 });
|
|
|
|
expect(res).toEqual({ result: 'fixed text' });
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://api.openai.com/v1/chat/completions');
|
|
expect(init.method).toBe('POST');
|
|
expect(init.headers).toEqual({ 'Content-Type': 'application/json', Authorization: 'Bearer sk-x' });
|
|
expect(JSON.parse(init.body)).toEqual({
|
|
model: 'gpt-4o-mini',
|
|
messages: [
|
|
{ role: 'system', content: 'SYS' },
|
|
{ role: 'user', content: 'hello' },
|
|
],
|
|
max_completion_tokens: 1024,
|
|
temperature: 0.7,
|
|
});
|
|
});
|
|
|
|
it('OpenAI reasoning models (o-series / gpt-5): max_completion_tokens, NO temperature', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
await callProvider({ provider: 'openai', apiKey: 'sk-x', model: 'gpt-5-mini' }, 'hello', 'SYS', { maxTokens: 1024 });
|
|
|
|
const body = JSON.parse(fetch.mock.calls[0][1].body);
|
|
expect(body.max_completion_tokens).toBe(1024);
|
|
expect(body.max_tokens).toBeUndefined();
|
|
expect(body.temperature).toBeUndefined();
|
|
});
|
|
|
|
it('Anthropic: x-api-key + version headers, top-level system, no temperature', async () => {
|
|
const fetch = mockFetchOnce(anthropicResponse);
|
|
const res = await callProvider({ provider: 'anthropic', apiKey: 'sk-ant' }, 'hello', 'SYS', { maxTokens: 1024 });
|
|
|
|
expect(res).toEqual({ result: 'fixed text' });
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://api.anthropic.com/v1/messages');
|
|
expect(init.headers).toEqual({
|
|
'Content-Type': 'application/json',
|
|
'x-api-key': 'sk-ant',
|
|
'anthropic-version': '2023-06-01',
|
|
'anthropic-dangerous-direct-browser-access': 'true',
|
|
});
|
|
expect(JSON.parse(init.body)).toEqual({
|
|
model: 'claude-3-5-haiku-20241022',
|
|
max_tokens: 1024,
|
|
system: 'SYS',
|
|
messages: [{ role: 'user', content: 'hello' }],
|
|
});
|
|
});
|
|
|
|
it('Groq: OpenAI-compatible endpoint and body with temperature', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
await callProvider({ provider: 'groq', apiKey: 'gsk-x' }, 'hello', 'SYS', { maxTokens: 1024 });
|
|
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://api.groq.com/openai/v1/chat/completions');
|
|
expect(init.headers).toEqual({ 'Content-Type': 'application/json', Authorization: 'Bearer gsk-x' });
|
|
const body = JSON.parse(init.body);
|
|
expect(body.model).toBe('llama-3.3-70b-versatile');
|
|
expect(body.temperature).toBe(0.7);
|
|
expect(body.max_tokens).toBe(1024);
|
|
});
|
|
|
|
it('OpenRouter: referer/title headers, NO temperature', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
await callProvider({ provider: 'openrouter', apiKey: 'sk-or' }, 'hello', 'SYS', { maxTokens: 1024 });
|
|
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://openrouter.ai/api/v1/chat/completions');
|
|
expect(init.headers).toEqual({
|
|
'Content-Type': 'application/json',
|
|
Authorization: 'Bearer sk-or',
|
|
'HTTP-Referer': 'https://lexai.dev',
|
|
'X-Title': 'LexAI',
|
|
});
|
|
const body = JSON.parse(init.body);
|
|
expect(body.model).toBe('openai/gpt-4o-mini');
|
|
expect(body).not.toHaveProperty('temperature');
|
|
});
|
|
|
|
it('uses the configured model over the default', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
await callProvider({ provider: 'openai', apiKey: 'k', model: 'gpt-4o' }, 'x', 'SYS');
|
|
expect(JSON.parse(fetch.mock.calls[0][1].body).model).toBe('gpt-4o');
|
|
});
|
|
});
|
|
|
|
describe('callProvider error handling (parity with old implementations)', () => {
|
|
it('surfaces provider error messages with the provider label', async () => {
|
|
// 401/403 additionally carry the settings hint — see 'key-rejection messaging'.
|
|
mockFetchOnce({ error: { message: 'rate limited' } }, { ok: false, status: 429 });
|
|
const res = await callProvider({ provider: 'openai', apiKey: 'bad' }, 'x', 'SYS');
|
|
expect(res).toEqual({ error: 'OpenAI error: rate limited' });
|
|
});
|
|
|
|
it('falls back to HTTP status when the error body has no message', async () => {
|
|
mockFetchOnce({}, { ok: false, status: 500 });
|
|
const res = await callProvider({ provider: 'groq', apiKey: 'k' }, 'x', 'SYS');
|
|
expect(res).toEqual({ error: 'Groq error: HTTP 500' });
|
|
});
|
|
|
|
it('reports network failures with the provider label', async () => {
|
|
vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline')));
|
|
const res = await callProvider({ provider: 'anthropic', apiKey: 'k' }, 'x', 'SYS');
|
|
expect(res.error).toMatch(/^Network error reaching Anthropic:/);
|
|
});
|
|
|
|
it('reports empty responses', async () => {
|
|
mockFetchOnce({ choices: [] });
|
|
const res = await callProvider({ provider: 'openrouter', apiKey: 'k' }, 'x', 'SYS');
|
|
expect(res).toEqual({ error: 'OpenRouter returned an empty response.' });
|
|
});
|
|
|
|
it('returns a clean HTTP error when the error body is not JSON (e.g. HTML 502)', async () => {
|
|
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
ok: false,
|
|
status: 502,
|
|
json: async () => { throw new SyntaxError('Unexpected token < in JSON'); },
|
|
}));
|
|
const res = await callProvider({ provider: 'openai', apiKey: 'k' }, 'x', 'SYS');
|
|
expect(res).toEqual({ error: 'OpenAI error: HTTP 502' });
|
|
});
|
|
|
|
it('rejects unknown providers without fetching', async () => {
|
|
const fetch = mockFetchOnce({});
|
|
const res = await callProvider({ provider: 'bogus', apiKey: 'k' }, 'x', 'SYS');
|
|
expect(res.error).toContain('Unknown provider: "bogus"');
|
|
expect(fetch).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('defaults to openai when no provider is configured', async () => {
|
|
const fetch = mockFetchOnce(openAiResponse);
|
|
await callProvider({ apiKey: 'k' }, 'x', 'SYS');
|
|
expect(fetch.mock.calls[0][0]).toBe('https://api.openai.com/v1/chat/completions');
|
|
});
|
|
});
|
|
|
|
describe('getSystemPrompt', () => {
|
|
it("normalizes 'fix' to the grammar prompt", () => {
|
|
expect(getSystemPrompt('fix')).toBe(getSystemPrompt('grammar'));
|
|
expect(getSystemPrompt('fix')).toContain('grammar editor');
|
|
});
|
|
|
|
it('falls back to grammar for unknown actions', () => {
|
|
expect(getSystemPrompt('nonsense')).toBe(getSystemPrompt('grammar'));
|
|
});
|
|
|
|
it('appends a style modifier except for Default', () => {
|
|
expect(getSystemPrompt('rephrase', 'Formal')).toMatch(/Write in a formal style\.$/);
|
|
expect(getSystemPrompt('rephrase', 'Default')).not.toContain('style.');
|
|
expect(getSystemPrompt('rephrase')).not.toContain('Write in a');
|
|
});
|
|
|
|
it("'prompt' uses the prompt-engineer base prompt plus the auto routing rubric", () => {
|
|
const base = getSystemPrompt('prompt');
|
|
expect(base).toContain('expert prompt engineer');
|
|
expect(base).toContain('Prefer the cheapest pattern that meets the goal');
|
|
// 'Default' style and an explicit all-Auto param set both add nothing.
|
|
expect(getSystemPrompt('prompt', 'Default')).toBe(base);
|
|
expect(getSystemPrompt('prompt', undefined, { pattern: 'auto', persona: 'Auto', format: 'Auto' })).toBe(base);
|
|
});
|
|
|
|
it("'prompt' appends a prompt-directed (not response-directed) style modifier, invariants last", () => {
|
|
const styled = getSystemPrompt('prompt', 'Formal');
|
|
expect(styled).toContain('instruct the model to respond in a formal style');
|
|
expect(styled).not.toContain('Write in a');
|
|
// Invariants ("Return ONLY the engineered prompt…") come after the style
|
|
// modifier, not before it — hard invariants are composed last.
|
|
expect(styled.indexOf('instruct the model to respond in a formal style'))
|
|
.toBeLessThan(styled.indexOf('Return ONLY the engineered prompt'));
|
|
});
|
|
|
|
// One distinctive, non-overlapping marker per PROMPT_PATTERNS id (except
|
|
// 'auto', which uses the routing rubric instead of a pattern instruction).
|
|
const PATTERN_MARKERS: Record<string, string[]> = {
|
|
'zero-shot': ['imperative instructions'],
|
|
role: ['ROLE, DOMAIN, INVARIANT, and OUTPUT_FORMAT'],
|
|
'few-shot': ['<example>', '<target>'],
|
|
structured: ['Context, Task, Constraints, Output format'],
|
|
contract: ['exact output schema'],
|
|
cot: ['PHASE 1: REASONING', 'PHASE 2: OUTPUT'],
|
|
'plan-solve': ['numbered plan'],
|
|
tot: ['score each 0-1'],
|
|
react: ['Final Answer:', 'step budget'],
|
|
pev: ['task DAG'],
|
|
gauntlet: ['STATUS: [PASS|FAIL]'],
|
|
};
|
|
|
|
it('every non-auto PROMPT_PATTERNS id injects its own marker and no other pattern\'s', () => {
|
|
const ids = Object.keys(PATTERN_MARKERS);
|
|
expect(ids.sort()).toEqual(
|
|
PROMPT_PATTERNS.filter((p) => p.id !== 'auto').map((p) => p.id).sort(),
|
|
);
|
|
|
|
for (const id of ids) {
|
|
const prompt = getSystemPrompt('prompt', undefined, { pattern: id });
|
|
for (const marker of PATTERN_MARKERS[id]) {
|
|
expect(prompt).toContain(marker);
|
|
}
|
|
for (const otherId of ids) {
|
|
if (otherId === id) continue;
|
|
for (const otherMarker of PATTERN_MARKERS[otherId]) {
|
|
expect(prompt).not.toContain(otherMarker);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it('an unknown or absent pattern falls back to the auto routing rubric', () => {
|
|
const base = getSystemPrompt('prompt');
|
|
expect(getSystemPrompt('prompt', undefined, { pattern: 'not-a-real-pattern' })).toBe(base);
|
|
expect(getSystemPrompt('prompt', undefined, {})).toBe(base);
|
|
});
|
|
|
|
it('Prompt Builder persona/format modifiers compose; Auto adds nothing', () => {
|
|
const full = getSystemPrompt('prompt', undefined, {
|
|
pattern: 'few-shot',
|
|
persona: 'Data Analyst',
|
|
format: 'JSON',
|
|
});
|
|
expect(full).toContain('<example>');
|
|
expect(full).toContain('persona of Data Analyst');
|
|
expect(full).toContain('final output as json');
|
|
|
|
expect(getSystemPrompt('prompt', undefined, { persona: 'None' })).toContain('Do not assign a persona');
|
|
// Params are prompt-action-only — other actions ignore them.
|
|
expect(getSystemPrompt('rephrase', undefined, { persona: 'Teacher' })).toBe(getSystemPrompt('rephrase'));
|
|
});
|
|
});
|
|
|
|
describe('resolvePromptPattern', () => {
|
|
const LEGACY_CASES: [string, string][] = [
|
|
['Auto', 'auto'],
|
|
['Instructional', 'zero-shot'],
|
|
['Role-play', 'role'],
|
|
['Step-by-step', 'cot'],
|
|
['Few-shot', 'few-shot'],
|
|
['Structured', 'structured'],
|
|
];
|
|
|
|
it('maps every legacy promptStyle label to its new pattern id', () => {
|
|
for (const [legacy, id] of LEGACY_CASES) {
|
|
expect(resolvePromptPattern(undefined, legacy)).toBe(id);
|
|
}
|
|
});
|
|
|
|
it('prefers an already-valid pattern id over a legacy style', () => {
|
|
expect(resolvePromptPattern('react', 'Few-shot')).toBe('react');
|
|
});
|
|
|
|
it('falls back to auto for unknown or absent input', () => {
|
|
expect(resolvePromptPattern(undefined, undefined)).toBe('auto');
|
|
expect(resolvePromptPattern('not-a-pattern', undefined)).toBe('auto');
|
|
expect(resolvePromptPattern(undefined, 'Not A Legacy Label')).toBe('auto');
|
|
});
|
|
});
|
|
|
|
describe('defaultMaxTokens', () => {
|
|
it('never goes below the old 1024 budget', () => {
|
|
expect(defaultMaxTokens('short')).toBe(1024);
|
|
});
|
|
|
|
it('scales with input length and clamps at 8192', () => {
|
|
expect(defaultMaxTokens('a'.repeat(4000))).toBe(4000);
|
|
expect(defaultMaxTokens('a'.repeat(50000))).toBe(8192);
|
|
});
|
|
});
|
|
|
|
describe('key-rejection messaging', () => {
|
|
it('appends a settings hint to 401/403 chat errors only', async () => {
|
|
mockFetchOnce({ error: { message: 'Invalid API Key' } }, { ok: false, status: 401 });
|
|
const rejected = await callProvider({ provider: 'groq', apiKey: 'bad' }, 'hi', 'SYS');
|
|
expect(rejected.error).toBe(
|
|
'Groq error: Invalid API Key — open LexAI Settings and re-enter your API key for this provider.',
|
|
);
|
|
|
|
mockFetchOnce({ error: { message: 'server exploded' } }, { ok: false, status: 500 });
|
|
const other = await callProvider({ provider: 'groq', apiKey: 'k' }, 'hi', 'SYS');
|
|
expect(other.error).toBe('Groq error: server exploded');
|
|
});
|
|
|
|
it('maps provider ids to display names', () => {
|
|
expect(providerLabel('groq')).toBe('Groq');
|
|
expect(providerLabel('anthropic')).toBe('Anthropic');
|
|
expect(providerLabel('mystery')).toBe('mystery');
|
|
});
|
|
});
|
|
|
|
describe('listModels', () => {
|
|
it('requires a key for Anthropic and sends the direct-browser-access header', async () => {
|
|
expect(await listModels('anthropic')).toEqual({
|
|
error: 'Anthropic requires an API key to list models.',
|
|
});
|
|
|
|
const fetch = mockFetchOnce({ data: [{ id: 'claude-3-5-haiku-20241022' }] });
|
|
await listModels('anthropic', 'sk-ant');
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://api.anthropic.com/v1/models');
|
|
expect(init.headers['anthropic-dangerous-direct-browser-access']).toBe('true');
|
|
expect(init.headers['x-api-key']).toBe('sk-ant');
|
|
});
|
|
|
|
it('allows keyless listing (OpenRouter) and sends bearer auth when a key exists', async () => {
|
|
const noKey = mockFetchOnce({ data: [{ id: 'openai/gpt-4o' }] });
|
|
await listModels('openrouter');
|
|
expect(noKey.mock.calls[0][1].headers).not.toHaveProperty('Authorization');
|
|
|
|
const withKey = mockFetchOnce({ data: [{ id: 'gpt-4o' }] });
|
|
await listModels('openai', 'sk-x');
|
|
expect(withKey.mock.calls[0][1].headers['Authorization']).toBe('Bearer sk-x');
|
|
});
|
|
|
|
it('filters non-chat models and sorts ids', async () => {
|
|
mockFetchOnce({
|
|
data: [
|
|
{ id: 'gpt-4o' },
|
|
{ id: 'text-embedding-3-small' },
|
|
{ id: 'whisper-1' },
|
|
{ id: 'dall-e-3' },
|
|
{ id: 'gpt-4o-mini' },
|
|
],
|
|
});
|
|
expect(await listModels('openai', 'k')).toEqual({ models: ['gpt-4o', 'gpt-4o-mini'] });
|
|
});
|
|
|
|
it('sends bearer auth for Groq and labels its errors with the display name', async () => {
|
|
const fetch = mockFetchOnce({ data: [{ id: 'llama-3.3-70b-versatile' }] });
|
|
expect(await listModels('groq', 'gsk_test')).toEqual({ models: ['llama-3.3-70b-versatile'] });
|
|
const [url, init] = fetch.mock.calls[0];
|
|
expect(url).toBe('https://api.groq.com/openai/v1/models');
|
|
expect(init.headers['Authorization']).toBe('Bearer gsk_test');
|
|
|
|
mockFetchOnce({ error: { message: 'Invalid API Key' } }, { ok: false, status: 401 });
|
|
expect((await listModels('groq', 'bad')).error).toBe('Groq error: Invalid API Key');
|
|
});
|
|
|
|
it('flags a rejected key so Options can prompt for a new one', async () => {
|
|
mockFetchOnce({ error: { message: 'Invalid API Key' } }, { ok: false, status: 401 });
|
|
expect(await listModels('groq', 'bad')).toEqual({
|
|
error: 'Groq error: Invalid API Key',
|
|
keyRejected: true,
|
|
});
|
|
|
|
mockFetchOnce({ error: { message: 'boom' } }, { ok: false, status: 500 });
|
|
expect((await listModels('groq', 'k')).keyRejected).toBeUndefined();
|
|
});
|
|
|
|
it('errors on empty lists and unknown providers', async () => {
|
|
mockFetchOnce({ data: [] });
|
|
expect((await listModels('openai', 'k')).error).toBe('No models returned by OpenAI.');
|
|
expect((await listModels('bogus', 'k')).error).toContain('Unknown provider');
|
|
});
|
|
});
|