Add new agents and skills for enhanced project orchestration and review processes
- Introduced `critic`, an independent adversarial reviewer for security and correctness. - Added `fable-orchestrator` to manage task routing and verification. - Implemented `gauntlet-critic` for fresh-context evaluation of gauntlet rounds. - Created `planner` for generating executable implementation plans with dependencies. - Developed `security-auditor` for application security reviews and audits. - Established `system-steward` to improve agent prompts and skills based on verified failures. - Added `dev-loop` skill for autonomous development loops over repositories. - Implemented `gauntlet-loop` skill for iterative quality benchmarking against reference standards. - Updated project settings to utilize the new orchestrator agent. - Created documentation for `GAUNTLET.md`, `PROGRESS.md`, and `REFERENCE_BAR.md` to track project status and quality benchmarks. - Added detailed prompting style guide to enhance understanding of prompt patterns and agentic loops.
This commit is contained in:
john kevin asprec
@@ -26,8 +26,49 @@ export const CONTEXT_MENU_STYLES = WRITING_STYLES.filter((s) => s !== 'Default')
|
||||
// ─── Prompt Builder parameters (the 'prompt' action) ─────────────────────────
|
||||
// 'Auto' always means "let the prompt engineer decide from the input".
|
||||
|
||||
export const PROMPT_STYLES = ['Auto', 'Instructional', 'Role-play', 'Step-by-step', 'Few-shot', 'Structured'] as const;
|
||||
export type PromptStyle = (typeof PROMPT_STYLES)[number];
|
||||
// Structural prompt-engineering patterns, drawn from docs/prompting_style.md.
|
||||
// Stable ids (not labels) are stored/persisted so relabeling stays free.
|
||||
// 'auto' has no group — it renders first, outside the optgroups.
|
||||
export interface PromptPatternDef {
|
||||
id: string;
|
||||
label: string;
|
||||
group: 'Direct' | 'Reasoning' | 'Agentic';
|
||||
hint: string; // one line: when to use it + its cost
|
||||
}
|
||||
|
||||
export const PROMPT_PATTERNS: PromptPatternDef[] = [
|
||||
{ id: 'auto', label: 'Auto', group: 'Direct', hint: 'Let the prompt engineer pick the cheapest pattern that fits the input.' },
|
||||
{ id: 'zero-shot', label: 'Zero-shot Instruction', group: 'Direct', hint: 'Direct imperative instructions, no scaffolding. Lowest cost — simple, single-step tasks.' },
|
||||
{ id: 'role', label: 'Role Conditioning', group: 'Direct', hint: 'Role/domain/invariants block; steadies tone and expertise across a longer chat.' },
|
||||
{ id: 'few-shot', label: 'Few-shot Examples', group: 'Direct', hint: 'Shows 1-2 example input→output pairs. Best for consistent formatting or extraction.' },
|
||||
{ id: 'structured', label: 'Structured Sections', group: 'Direct', hint: 'Labeled Context/Task/Constraints/Output. Clearer for multi-constraint requests.' },
|
||||
{ id: 'contract', label: 'Output Contract', group: 'Direct', hint: 'Pins an exact output schema. Best when downstream code parses the result.' },
|
||||
{ id: 'cot', label: 'Chain-of-Thought', group: 'Reasoning', hint: 'Reasons step by step before answering. For math, logic, multi-constraint problems.' },
|
||||
{ id: 'plan-solve', label: 'Plan-and-Solve', group: 'Reasoning', hint: 'Plans first, then executes in order. For open-ended design or multi-part tasks.' },
|
||||
{ id: 'tot', label: 'Tree-of-Thoughts', group: 'Reasoning', hint: 'Scores multiple candidate approaches and keeps the best. High cost — hard planning/refactors.' },
|
||||
{ id: 'react', label: 'ReAct (tools)', group: 'Agentic', hint: 'Thought/Action/Observation tool loop for a tool-capable agent, not a plain chat.' },
|
||||
{ id: 'pev', label: 'Plan-Execute-Verify', group: 'Agentic', hint: 'Task DAG with per-step verification, for a tool-capable agent on multi-step builds.' },
|
||||
{ id: 'gauntlet', label: 'Gauntlet (Builder–Judge)', group: 'Agentic', hint: 'Builder vs. fresh-context judge on a named standard, for a tool-capable agent. Very high cost.' },
|
||||
];
|
||||
|
||||
// Legacy label -> new pattern id (storage persists across versions).
|
||||
const LEGACY_PATTERN_IDS: Record<string, string> = {
|
||||
Auto: 'auto',
|
||||
Instructional: 'zero-shot',
|
||||
'Role-play': 'role',
|
||||
'Step-by-step': 'cot',
|
||||
'Few-shot': 'few-shot',
|
||||
Structured: 'structured',
|
||||
};
|
||||
|
||||
// Resolves a stored Prompt Builder pattern selection into a valid
|
||||
// PROMPT_PATTERNS id: `pattern` if already a known id, else the legacy
|
||||
// `promptStyle` label mapped through LEGACY_PATTERN_IDS, else 'auto'.
|
||||
export function resolvePromptPattern(pattern?: string, legacyStyle?: string): string {
|
||||
if (pattern && PROMPT_PATTERNS.some((p) => p.id === pattern)) return pattern;
|
||||
if (legacyStyle && LEGACY_PATTERN_IDS[legacyStyle]) return LEGACY_PATTERN_IDS[legacyStyle];
|
||||
return 'auto';
|
||||
}
|
||||
|
||||
// 'Custom…' switches the popup to a free-text persona field.
|
||||
export const PROMPT_PERSONAS = [
|
||||
|
||||
@@ -22,24 +22,81 @@ export async function fetchWithTimeout(
|
||||
|
||||
// ─── System prompts ───────────────────────────────────────────────────────────
|
||||
|
||||
// Prompt Builder parameters → extra instructions for the prompt-engineer
|
||||
// action. 'Auto' (or absence) adds nothing — the base prompt already tells the
|
||||
// engineer to decide these from the input.
|
||||
// ─── Prompt Builder pattern library (docs/prompting_style.md) ────────────────
|
||||
// Composed in a fixed order — ROLE+TASK, PATTERN, MODIFIERS, INVARIANTS last
|
||||
// (invariants last so they survive context decay, guide §1) — never the whole
|
||||
// table at once: only the selected pattern's instruction+skeleton+guard.
|
||||
|
||||
const PROMPT_BASE_TASK =
|
||||
'You are an expert prompt engineer. Transform the provided text into one well-crafted prompt that will get the best possible result from an AI model. ' +
|
||||
'First infer what the user is trying to achieve — the text may be a rough idea, a question, or a description of the output they want — then compose exactly one engineered prompt for that goal.';
|
||||
|
||||
// Auto routing rubric — lives in the base so 'auto' (or an absent/unknown
|
||||
// pattern) adds nothing beyond it. Mirrors the guide's summary matrix.
|
||||
const PROMPT_AUTO_RUBRIC =
|
||||
' Choose the structural pattern the goal actually needs: formatting or extraction favors few-shot examples or an output contract; ' +
|
||||
'logic or multi-constraint problems favor chain-of-thought; open-ended design favors plan-and-solve or tree-of-thoughts; ' +
|
||||
'tasks needing live data or tools favor ReAct; multi-step builds or migrations favor plan-execute-verify; ' +
|
||||
'tasks that must beat a quality bar favor a gauntlet builder-judge loop; otherwise use a plain zero-shot instruction. ' +
|
||||
'Prefer the cheapest pattern that meets the goal — never add reasoning scaffolding to a simple task.';
|
||||
|
||||
// One instruction+skeleton+failure-mode-guard per PROMPT_PATTERNS id
|
||||
// (excluding 'auto', which uses the rubric above instead).
|
||||
const PROMPT_PATTERN_INSTRUCTIONS: Record<string, string> = {
|
||||
'zero-shot':
|
||||
' Compose the engineered prompt as direct, imperative instructions — one clear task per sentence, ' +
|
||||
'with no scaffolding beyond what the task actually needs.',
|
||||
role:
|
||||
' Compose the engineered prompt as a system-role block with ROLE, DOMAIN, INVARIANT, and OUTPUT_FORMAT lines that assign the model ' +
|
||||
'its expertise and constraints; restate the single hardest constraint again as the very last line, to guard against it being forgotten ' +
|
||||
'in a long conversation (context decay).',
|
||||
'few-shot':
|
||||
' Compose the engineered prompt using explicit <example><input>…</input><output>…</output></example> delimiters, with 1-2 examples ' +
|
||||
'that are structurally diverse from each other (not near-duplicates), to guard against the model overfitting to the last example\'s ' +
|
||||
'exact values (recency/label bias), then a trailing open <target><input>…</input><output> for the real input.',
|
||||
structured:
|
||||
' Compose the engineered prompt as labeled sections in this order: Context, Task, Constraints, Output format — each a short heading ' +
|
||||
'followed by its content.',
|
||||
contract:
|
||||
' Compose the engineered prompt around an exact output schema: name every field, its type, and whether it is required, and include a ' +
|
||||
'rule instructing the model to reject or omit anything outside that schema.',
|
||||
cot:
|
||||
' Compose the engineered prompt with two explicit phases labeled PHASE 1: REASONING and PHASE 2: OUTPUT — the model works through its ' +
|
||||
'reasoning in phase 1, then gives a final answer in phase 2 that stands alone without needing the reasoning to make sense.',
|
||||
'plan-solve':
|
||||
' Compose the engineered prompt so the model must first produce a numbered plan of the steps needed, then execute that plan in order, ' +
|
||||
'referencing each step as it completes it.',
|
||||
tot:
|
||||
' Compose the engineered prompt instructing the model to generate several candidate approaches, score each 0-1 against stated criteria, ' +
|
||||
'prune the weak ones, expand on the best, and report the winning approach and why it was chosen.',
|
||||
react:
|
||||
' Compose the engineered prompt for a tool-capable agent, not a plain chat model: declare the available tools, require a strict ' +
|
||||
'Thought: / Action: / Observation: loop, and terminate with a line starting Final Answer:. Include a hard step budget (e.g. max 10 steps) ' +
|
||||
'and a rule that repeating the same action signature twice must break the loop, to guard against infinite loops.',
|
||||
pev:
|
||||
' Compose the engineered prompt for a tool-capable agent, not a plain chat model: require it to first generate a task DAG of sub-tasks, ' +
|
||||
'run a verification assertion after each node, re-plan the remaining DAG on any failed assertion, and state an explicit stop condition ' +
|
||||
'for when the task is complete.',
|
||||
gauntlet:
|
||||
' Compose the engineered prompt for a tool-capable agent, not a plain chat model, running a builder-judge loop: the builder produces an ' +
|
||||
'artifact, a judge instantiated in a fresh context compares it against a NAMED reference standard, and returns exactly ' +
|
||||
'STATUS: [PASS|FAIL] | FEEDBACK: <one directive>; the loop stops on PASS or after a stated maximum number of rounds.',
|
||||
};
|
||||
|
||||
function promptPatternSection(pattern?: string): string {
|
||||
if (pattern && pattern !== 'auto' && PROMPT_PATTERN_INSTRUCTIONS[pattern]) {
|
||||
return PROMPT_PATTERN_INSTRUCTIONS[pattern];
|
||||
}
|
||||
return PROMPT_AUTO_RUBRIC;
|
||||
}
|
||||
|
||||
// Persona + output-format modifiers (Prompt Builder parameters). 'Auto'/absent
|
||||
// adds nothing for either — the base prompt already tells the engineer to
|
||||
// decide these from the input.
|
||||
function promptParamModifiers(params?: PromptParams): string {
|
||||
if (!params) return '';
|
||||
const parts: string[] = [];
|
||||
|
||||
const styleInstructions: Record<string, string> = {
|
||||
Instructional: ' Compose the engineered prompt as clear, direct instructions.',
|
||||
'Role-play': ' Compose the engineered prompt as a role-play scenario the model should stay in character for.',
|
||||
'Step-by-step': ' The engineered prompt should ask the model to work through the task step by step before giving its final answer.',
|
||||
'Few-shot': ' Include one or two short input→output examples (few-shot) in the engineered prompt.',
|
||||
Structured: ' Organize the engineered prompt into labeled sections (Context, Task, Constraints, Output format).',
|
||||
};
|
||||
if (params.promptStyle && styleInstructions[params.promptStyle]) {
|
||||
parts.push(styleInstructions[params.promptStyle]);
|
||||
}
|
||||
|
||||
if (params.persona === 'None') {
|
||||
parts.push(' Do not assign a persona or role in the engineered prompt.');
|
||||
} else if (params.persona && params.persona !== 'Auto') {
|
||||
@@ -53,6 +110,13 @@ function promptParamModifiers(params?: PromptParams): string {
|
||||
return parts.join('');
|
||||
}
|
||||
|
||||
// Hard invariants — always last, so they survive context decay on long
|
||||
// compositions (guide §1: restate/keep critical rules at the end).
|
||||
const PROMPT_INVARIANTS =
|
||||
' Return ONLY the engineered prompt, ready to paste into an AI chat — no explanations, no surrounding quotes, no preamble. ' +
|
||||
'Keep it self-contained, and use the cheapest structure that meets the goal. ' +
|
||||
'If the input is missing information the prompt needs, mark it as a [BRACKETED] placeholder rather than inventing facts.';
|
||||
|
||||
export function getSystemPrompt(action: string, style?: string, promptParams?: PromptParams): string {
|
||||
// Normalize 'fix' (used by context menu and popup) to 'grammar'
|
||||
const normalizedAction = action === 'fix' ? 'grammar' : action;
|
||||
@@ -77,23 +141,26 @@ export function getSystemPrompt(action: string, style?: string, promptParams?: P
|
||||
'You are a helpful teacher. Explain the following text in simple, easy-to-understand language. ' +
|
||||
'Break down complex terms, jargon, or concepts so anyone can understand. ' +
|
||||
'Be concise but clear. Return only the explanation, no extra commentary.',
|
||||
prompt:
|
||||
'You are an expert prompt engineer. Transform the provided text into one well-crafted prompt that will get the best possible result from an AI model. ' +
|
||||
'First infer what the user is trying to achieve — the text may be a rough idea, a question, or a description of the output they want. ' +
|
||||
'Then compose the prompt using only the components that goal actually needs: a persona/role and relevant skills when expertise helps, essential context, a clear task statement, constraints, and the desired output format or structure. ' +
|
||||
'Make the prompt specific, self-contained, and unambiguous, and phrase it as instructions addressed to an AI model. ' +
|
||||
'Return ONLY the engineered prompt, ready to paste into an AI chat — no explanations, no surrounding quotes, no preamble.',
|
||||
prompt: PROMPT_BASE_TASK,
|
||||
};
|
||||
const base = prompts[normalizedAction] ?? prompts.grammar;
|
||||
// For the prompt-engineer action, the style describes the output the
|
||||
// engineered prompt should ask for — not the wording of our response.
|
||||
|
||||
if (normalizedAction !== 'prompt') {
|
||||
// For every other action, style describes the wording of OUR response.
|
||||
const styleModifier = style && style !== 'Default' ? ` Write in a ${style.toLowerCase()} style.` : '';
|
||||
return base + styleModifier;
|
||||
}
|
||||
|
||||
// 'prompt' action: ROLE+TASK, then PATTERN, then MODIFIERS (persona,
|
||||
// format, response style — in that order), then hard INVARIANTS last.
|
||||
const patternSection = promptPatternSection(promptParams?.pattern);
|
||||
const modifiers = promptParamModifiers(promptParams);
|
||||
// The style here describes the output the ENGINEERED prompt should ask
|
||||
// for — not the wording of this system prompt.
|
||||
const styleModifier = style && style !== 'Default'
|
||||
? normalizedAction === 'prompt'
|
||||
? ` The engineered prompt should instruct the model to respond in a ${style.toLowerCase()} style.`
|
||||
: ` Write in a ${style.toLowerCase()} style.`
|
||||
? ` The engineered prompt should instruct the model to respond in a ${style.toLowerCase()} style.`
|
||||
: '';
|
||||
const paramModifiers = normalizedAction === 'prompt' ? promptParamModifiers(promptParams) : '';
|
||||
return base + styleModifier + paramModifiers;
|
||||
return base + patternSection + modifiers + styleModifier + PROMPT_INVARIANTS;
|
||||
}
|
||||
|
||||
// ─── Adapter table ────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
// Extra parameters for the 'prompt' (prompt-engineer) action. All optional;
|
||||
// omitted / 'Auto' means the prompt engineer decides from the input itself.
|
||||
export interface PromptParams {
|
||||
promptStyle?: string; // see PROMPT_STYLES in actions.ts
|
||||
persona?: string; // preset from PROMPT_PERSONAS, or free text (Custom)
|
||||
format?: string; // see PROMPT_FORMATS
|
||||
pattern?: string; // see PROMPT_PATTERNS in actions.ts
|
||||
persona?: string; // preset from PROMPT_PERSONAS, or free text (Custom)
|
||||
format?: string; // see PROMPT_FORMATS
|
||||
}
|
||||
|
||||
export interface AnalyzePayload {
|
||||
|
||||
Reference in New Issue
Block a user