Add new agents and skills for enhanced project orchestration and review processes
- Introduced `critic`, an independent adversarial reviewer for security and correctness. - Added `fable-orchestrator` to manage task routing and verification. - Implemented `gauntlet-critic` for fresh-context evaluation of gauntlet rounds. - Created `planner` for generating executable implementation plans with dependencies. - Developed `security-auditor` for application security reviews and audits. - Established `system-steward` to improve agent prompts and skills based on verified failures. - Added `dev-loop` skill for autonomous development loops over repositories. - Implemented `gauntlet-loop` skill for iterative quality benchmarking against reference standards. - Updated project settings to utilize the new orchestrator agent. - Created documentation for `GAUNTLET.md`, `PROGRESS.md`, and `REFERENCE_BAR.md` to track project status and quality benchmarks. - Added detailed prompting style guide to enhance understanding of prompt patterns and agentic loops.
This commit is contained in:
@@ -7,6 +7,7 @@ import {
|
||||
providerLabel,
|
||||
PROVIDER_SPECS,
|
||||
} from '@lib/providers';
|
||||
import { PROMPT_PATTERNS, resolvePromptPattern } from '@lib/actions';
|
||||
|
||||
function mockFetchOnce(data: unknown, { ok = true, status = 200 } = {}) {
|
||||
const fn = vi.fn().mockResolvedValue({ ok, status, json: async () => data });
|
||||
@@ -179,23 +180,74 @@ describe('getSystemPrompt', () => {
|
||||
expect(getSystemPrompt('rephrase')).not.toContain('Write in a');
|
||||
});
|
||||
|
||||
it("'prompt' uses the prompt-engineer prompt with a prompt-directed style modifier", () => {
|
||||
expect(getSystemPrompt('prompt')).toContain('expert prompt engineer');
|
||||
expect(getSystemPrompt('prompt', 'Formal')).toMatch(/instruct the model to respond in a formal style\.$/);
|
||||
expect(getSystemPrompt('prompt', 'Formal')).not.toContain('Write in a');
|
||||
expect(getSystemPrompt('prompt', 'Default')).toBe(getSystemPrompt('prompt'));
|
||||
it("'prompt' uses the prompt-engineer base prompt plus the auto routing rubric", () => {
|
||||
const base = getSystemPrompt('prompt');
|
||||
expect(base).toContain('expert prompt engineer');
|
||||
expect(base).toContain('Prefer the cheapest pattern that meets the goal');
|
||||
// 'Default' style and an explicit all-Auto param set both add nothing.
|
||||
expect(getSystemPrompt('prompt', 'Default')).toBe(base);
|
||||
expect(getSystemPrompt('prompt', undefined, { pattern: 'auto', persona: 'Auto', format: 'Auto' })).toBe(base);
|
||||
});
|
||||
|
||||
it('Prompt Builder params add instructions; Auto adds nothing', () => {
|
||||
const base = getSystemPrompt('prompt');
|
||||
expect(getSystemPrompt('prompt', undefined, { promptStyle: 'Auto', persona: 'Auto', format: 'Auto' })).toBe(base);
|
||||
it("'prompt' appends a prompt-directed (not response-directed) style modifier, invariants last", () => {
|
||||
const styled = getSystemPrompt('prompt', 'Formal');
|
||||
expect(styled).toContain('instruct the model to respond in a formal style');
|
||||
expect(styled).not.toContain('Write in a');
|
||||
// Invariants ("Return ONLY the engineered prompt…") come after the style
|
||||
// modifier, not before it — hard invariants are composed last.
|
||||
expect(styled.indexOf('instruct the model to respond in a formal style'))
|
||||
.toBeLessThan(styled.indexOf('Return ONLY the engineered prompt'));
|
||||
});
|
||||
|
||||
// One distinctive, non-overlapping marker per PROMPT_PATTERNS id (except
|
||||
// 'auto', which uses the routing rubric instead of a pattern instruction).
|
||||
const PATTERN_MARKERS: Record<string, string[]> = {
|
||||
'zero-shot': ['imperative instructions'],
|
||||
role: ['ROLE, DOMAIN, INVARIANT, and OUTPUT_FORMAT'],
|
||||
'few-shot': ['<example>', '<target>'],
|
||||
structured: ['Context, Task, Constraints, Output format'],
|
||||
contract: ['exact output schema'],
|
||||
cot: ['PHASE 1: REASONING', 'PHASE 2: OUTPUT'],
|
||||
'plan-solve': ['numbered plan'],
|
||||
tot: ['score each 0-1'],
|
||||
react: ['Final Answer:', 'step budget'],
|
||||
pev: ['task DAG'],
|
||||
gauntlet: ['STATUS: [PASS|FAIL]'],
|
||||
};
|
||||
|
||||
it('every non-auto PROMPT_PATTERNS id injects its own marker and no other pattern\'s', () => {
|
||||
const ids = Object.keys(PATTERN_MARKERS);
|
||||
expect(ids.sort()).toEqual(
|
||||
PROMPT_PATTERNS.filter((p) => p.id !== 'auto').map((p) => p.id).sort(),
|
||||
);
|
||||
|
||||
for (const id of ids) {
|
||||
const prompt = getSystemPrompt('prompt', undefined, { pattern: id });
|
||||
for (const marker of PATTERN_MARKERS[id]) {
|
||||
expect(prompt).toContain(marker);
|
||||
}
|
||||
for (const otherId of ids) {
|
||||
if (otherId === id) continue;
|
||||
for (const otherMarker of PATTERN_MARKERS[otherId]) {
|
||||
expect(prompt).not.toContain(otherMarker);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('an unknown or absent pattern falls back to the auto routing rubric', () => {
|
||||
const base = getSystemPrompt('prompt');
|
||||
expect(getSystemPrompt('prompt', undefined, { pattern: 'not-a-real-pattern' })).toBe(base);
|
||||
expect(getSystemPrompt('prompt', undefined, {})).toBe(base);
|
||||
});
|
||||
|
||||
it('Prompt Builder persona/format modifiers compose; Auto adds nothing', () => {
|
||||
const full = getSystemPrompt('prompt', undefined, {
|
||||
promptStyle: 'Few-shot',
|
||||
pattern: 'few-shot',
|
||||
persona: 'Data Analyst',
|
||||
format: 'JSON',
|
||||
});
|
||||
expect(full).toContain('few-shot');
|
||||
expect(full).toContain('<example>');
|
||||
expect(full).toContain('persona of Data Analyst');
|
||||
expect(full).toContain('final output as json');
|
||||
|
||||
@@ -205,6 +257,33 @@ describe('getSystemPrompt', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolvePromptPattern', () => {
|
||||
const LEGACY_CASES: [string, string][] = [
|
||||
['Auto', 'auto'],
|
||||
['Instructional', 'zero-shot'],
|
||||
['Role-play', 'role'],
|
||||
['Step-by-step', 'cot'],
|
||||
['Few-shot', 'few-shot'],
|
||||
['Structured', 'structured'],
|
||||
];
|
||||
|
||||
it('maps every legacy promptStyle label to its new pattern id', () => {
|
||||
for (const [legacy, id] of LEGACY_CASES) {
|
||||
expect(resolvePromptPattern(undefined, legacy)).toBe(id);
|
||||
}
|
||||
});
|
||||
|
||||
it('prefers an already-valid pattern id over a legacy style', () => {
|
||||
expect(resolvePromptPattern('react', 'Few-shot')).toBe('react');
|
||||
});
|
||||
|
||||
it('falls back to auto for unknown or absent input', () => {
|
||||
expect(resolvePromptPattern(undefined, undefined)).toBe('auto');
|
||||
expect(resolvePromptPattern('not-a-pattern', undefined)).toBe('auto');
|
||||
expect(resolvePromptPattern(undefined, 'Not A Legacy Label')).toBe('auto');
|
||||
});
|
||||
});
|
||||
|
||||
describe('defaultMaxTokens', () => {
|
||||
it('never goes below the old 1024 budget', () => {
|
||||
expect(defaultMaxTokens('short')).toBe(1024);
|
||||
|
||||
Reference in New Issue
Block a user