Add new agents and skills for enhanced project orchestration and review processes

- Introduced `critic`, an independent adversarial reviewer for security and correctness.
- Added `fable-orchestrator` to manage task routing and verification.
- Implemented `gauntlet-critic` for fresh-context evaluation of gauntlet rounds.
- Created `planner` for generating executable implementation plans with dependencies.
- Developed `security-auditor` for application security reviews and audits.
- Established `system-steward` to improve agent prompts and skills based on verified failures.
- Added `dev-loop` skill for autonomous development loops over repositories.
- Implemented `gauntlet-loop` skill for iterative quality benchmarking against reference standards.
- Updated project settings to utilize the new orchestrator agent.
- Created documentation for `GAUNTLET.md`, `PROGRESS.md`, and `REFERENCE_BAR.md` to track project status and quality benchmarks.
- Added detailed prompting style guide to enhance understanding of prompt patterns and agentic loops.
This commit is contained in:
john kevin asprec
2026-08-08 16:49:07 +08:00
parent 6aee260533
commit 444060c3eb
85 changed files with 2717 additions and 171 deletions

View File

@@ -7,6 +7,7 @@ import {
providerLabel,
PROVIDER_SPECS,
} from '@lib/providers';
import { PROMPT_PATTERNS, resolvePromptPattern } from '@lib/actions';
function mockFetchOnce(data: unknown, { ok = true, status = 200 } = {}) {
const fn = vi.fn().mockResolvedValue({ ok, status, json: async () => data });
@@ -179,23 +180,74 @@ describe('getSystemPrompt', () => {
expect(getSystemPrompt('rephrase')).not.toContain('Write in a');
});
it("'prompt' uses the prompt-engineer prompt with a prompt-directed style modifier", () => {
expect(getSystemPrompt('prompt')).toContain('expert prompt engineer');
expect(getSystemPrompt('prompt', 'Formal')).toMatch(/instruct the model to respond in a formal style\.$/);
expect(getSystemPrompt('prompt', 'Formal')).not.toContain('Write in a');
expect(getSystemPrompt('prompt', 'Default')).toBe(getSystemPrompt('prompt'));
it("'prompt' uses the prompt-engineer base prompt plus the auto routing rubric", () => {
const base = getSystemPrompt('prompt');
expect(base).toContain('expert prompt engineer');
expect(base).toContain('Prefer the cheapest pattern that meets the goal');
// 'Default' style and an explicit all-Auto param set both add nothing.
expect(getSystemPrompt('prompt', 'Default')).toBe(base);
expect(getSystemPrompt('prompt', undefined, { pattern: 'auto', persona: 'Auto', format: 'Auto' })).toBe(base);
});
it('Prompt Builder params add instructions; Auto adds nothing', () => {
const base = getSystemPrompt('prompt');
expect(getSystemPrompt('prompt', undefined, { promptStyle: 'Auto', persona: 'Auto', format: 'Auto' })).toBe(base);
it("'prompt' appends a prompt-directed (not response-directed) style modifier, invariants last", () => {
const styled = getSystemPrompt('prompt', 'Formal');
expect(styled).toContain('instruct the model to respond in a formal style');
expect(styled).not.toContain('Write in a');
// Invariants ("Return ONLY the engineered prompt…") come after the style
// modifier, not before it — hard invariants are composed last.
expect(styled.indexOf('instruct the model to respond in a formal style'))
.toBeLessThan(styled.indexOf('Return ONLY the engineered prompt'));
});
// One distinctive, non-overlapping marker per PROMPT_PATTERNS id (except
// 'auto', which uses the routing rubric instead of a pattern instruction).
const PATTERN_MARKERS: Record<string, string[]> = {
'zero-shot': ['imperative instructions'],
role: ['ROLE, DOMAIN, INVARIANT, and OUTPUT_FORMAT'],
'few-shot': ['<example>', '<target>'],
structured: ['Context, Task, Constraints, Output format'],
contract: ['exact output schema'],
cot: ['PHASE 1: REASONING', 'PHASE 2: OUTPUT'],
'plan-solve': ['numbered plan'],
tot: ['score each 0-1'],
react: ['Final Answer:', 'step budget'],
pev: ['task DAG'],
gauntlet: ['STATUS: [PASS|FAIL]'],
};
it('every non-auto PROMPT_PATTERNS id injects its own marker and no other pattern\'s', () => {
const ids = Object.keys(PATTERN_MARKERS);
expect(ids.sort()).toEqual(
PROMPT_PATTERNS.filter((p) => p.id !== 'auto').map((p) => p.id).sort(),
);
for (const id of ids) {
const prompt = getSystemPrompt('prompt', undefined, { pattern: id });
for (const marker of PATTERN_MARKERS[id]) {
expect(prompt).toContain(marker);
}
for (const otherId of ids) {
if (otherId === id) continue;
for (const otherMarker of PATTERN_MARKERS[otherId]) {
expect(prompt).not.toContain(otherMarker);
}
}
}
});
it('an unknown or absent pattern falls back to the auto routing rubric', () => {
const base = getSystemPrompt('prompt');
expect(getSystemPrompt('prompt', undefined, { pattern: 'not-a-real-pattern' })).toBe(base);
expect(getSystemPrompt('prompt', undefined, {})).toBe(base);
});
it('Prompt Builder persona/format modifiers compose; Auto adds nothing', () => {
const full = getSystemPrompt('prompt', undefined, {
promptStyle: 'Few-shot',
pattern: 'few-shot',
persona: 'Data Analyst',
format: 'JSON',
});
expect(full).toContain('few-shot');
expect(full).toContain('<example>');
expect(full).toContain('persona of Data Analyst');
expect(full).toContain('final output as json');
@@ -205,6 +257,33 @@ describe('getSystemPrompt', () => {
});
});
describe('resolvePromptPattern', () => {
const LEGACY_CASES: [string, string][] = [
['Auto', 'auto'],
['Instructional', 'zero-shot'],
['Role-play', 'role'],
['Step-by-step', 'cot'],
['Few-shot', 'few-shot'],
['Structured', 'structured'],
];
it('maps every legacy promptStyle label to its new pattern id', () => {
for (const [legacy, id] of LEGACY_CASES) {
expect(resolvePromptPattern(undefined, legacy)).toBe(id);
}
});
it('prefers an already-valid pattern id over a legacy style', () => {
expect(resolvePromptPattern('react', 'Few-shot')).toBe('react');
});
it('falls back to auto for unknown or absent input', () => {
expect(resolvePromptPattern(undefined, undefined)).toBe('auto');
expect(resolvePromptPattern('not-a-pattern', undefined)).toBe('auto');
expect(resolvePromptPattern(undefined, 'Not A Legacy Label')).toBe('auto');
});
});
describe('defaultMaxTokens', () => {
it('never goes below the old 1024 budget', () => {
expect(defaultMaxTokens('short')).toBe(1024);