From a633c67538ec881f3c24092cfb5996f4c031b54c Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 5 May 2026 10:09:31 -0500 Subject: [PATCH] Refactor: split prompt definition from execution settings and migrate run contracts to prompt_* + execution_target --- README.md | 150 ++--- architecture.md | 234 ++++--- internal/adapter/cli/run.go | 54 +- internal/adapter/cli/run_test.go | 29 +- internal/adapter/http/dto.go | 50 +- internal/adapter/http/handler.go | 68 +- internal/adapter/http/handler_test.go | 45 +- internal/domain/domain.go | 141 +++-- internal/llm/openai_compatible_client.go | 12 +- internal/llm/openai_compatible_client_test.go | 42 +- internal/profile/filesystem_repository.go | 39 +- internal/profile/repository.go | 6 +- internal/profile/repository_test.go | 35 +- .../testdata/json_schema_missing_path.yaml | 3 + internal/profile/testdata/missing_id.yaml | 3 + .../profile/testdata/negative_timeout.yaml | 5 +- internal/profile/testdata/no_templates.yaml | 3 + internal/profile/testdata/valid.yaml | 17 +- internal/prompt/go_renderer.go | 20 +- internal/prompt/renderer.go | 2 +- internal/prompt/renderer_test.go | 16 +- internal/usecase/integration_test.go | 19 +- internal/usecase/repairer.go | 2 +- internal/usecase/runner.go | 95 ++- internal/usecase/runner_test.go | 591 +++--------------- profiles/dnd.session_recap.yaml | 18 +- profiles/generic.markdown_summary.yaml | 18 +- profiles/generic.structured_events.yaml | 16 +- 28 files changed, 712 insertions(+), 1021 deletions(-) diff --git a/README.md b/README.md index ea3c53d..2157750 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # scriptorium -Scriptorium is a generic prompt-profile execution engine written in Go. +Scriptorium is a generic prompt-definition execution engine written in Go. -Given named input artifacts and a prompt profile, Scriptorium: +Given named input artifacts and a prompt definition, Scriptorium: -1. Loads the profile. +1. Loads the prompt definition. 2. Resolves input artifact references. 3. Renders prompt messages from templates. 4. Calls an OpenAI-compatible LLM endpoint. @@ -28,36 +28,23 @@ D&D-specific behavior belongs in profiles, schemas, fixtures, and caller inputs, ## Core Concepts -- Prompt profile: YAML config that defines templates, model defaults, output format, and validation behavior. -- Named inputs: logical input names (for example `transcript`, `glossary`) mapped to artifact references. -- Artifact refs: currently `file` and `inline` are supported by readers used in v1 flows. -- Template variables: key/value vars provided at run time and accessed in templates as `{{.var_name}}`. -- Model target: endpoint/model and generation parameters (`temperature`, `max_tokens`, `top_p`, `timeout_seconds`). +- Prompt definition: YAML config for templates, inputs, output format, and validation behavior. +- Execution profile: conceptual runtime config (endpoint/model/timeouts/auth source). In this transition, execution settings are supplied as run-time overrides. +- Named inputs: logical names (for example `transcript`, `glossary`) mapped to artifact references. +- Artifact refs: currently `file` and `inline` are supported. +- Template variables: key/value vars passed at run time and referenced as `{{.var_name}}`. +- Execution target: endpoint/model plus generation/runtime parameters (`temperature`, `max_tokens`, `top_p`, `timeout_seconds`, `reasoning_effort`, `api_key_env`). - Output format: `text`, `markdown`, or `json`. - Validation mode: `none`, `basic`, `json`, `json_schema`. - Repair attempts: bounded retries for structured modes (`json`, `json_schema`) when output validation fails. -- Run metadata: IDs/hashes/model/timing/usage/validation details for auditability. ## Build and Test -Build: - ```bash go build -o scriptorium ./cmd/scriptorium -``` - -Run tests: - -```bash go test ./... ``` -Run CLI locally: - -```bash -go run ./cmd/scriptorium run --help -``` - ## CLI Usage ### `scriptorium run` @@ -65,57 +52,38 @@ go run ./cmd/scriptorium run --help Required flags: - `--profile-dir` -- `--profile-id` +- `--prompt-id` - `--input` (repeatable `name=path`) -Optional flags: +Common optional flags: +- `--profile-id` (execution profile selector; falls back to prompt `default_profile`) - `--var` (repeatable `name=value`) - `--out` - `--llm-base-url` -- `--llm-api-key` - `--model` +- `--api-key-env` - `--temperature` - `--max-tokens` - `--schema-dir` - `--timeout` -If `--llm-base-url` and/or `--model` are omitted, profile `model_defaults` must provide them. +Current transitional behavior: execution-profile loading is not implemented yet, so run-time execution settings must be supplied via overrides. In practice, provide at least endpoint and model (`--llm-base-url` and `--model`). -Markdown summary example: +Example: ```bash +export SCRIPTORIUM_API_KEY="your-key" + go run ./cmd/scriptorium run \ --profile-dir ./profiles \ - --profile-id generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml \ - --out ./out.md -``` - -Same run with explicit local OpenAI-compatible endpoint (for example vLLM): - -```bash -go run ./cmd/scriptorium run \ - --profile-dir ./profiles \ - --profile-id generic.markdown_summary \ + --prompt-id generic.markdown_summary \ + --profile-id local-default \ --input transcript=./examples/fixtures/transcript.md \ --input glossary=./examples/fixtures/glossary.yml \ --llm-base-url http://localhost:8000/v1 \ --model gpt-4o-mini \ - --out ./out.md -``` - -Passing template variables: - -```bash -go run ./cmd/scriptorium run \ - --profile-dir ./profiles \ - --profile-id generic.markdown_summary \ - --input transcript=./examples/fixtures/transcript.md \ - --input glossary=./examples/fixtures/glossary.yml \ - --var session_date=2026-05-04 \ - --var facilitator="Eris" \ + --api-key-env SCRIPTORIUM_API_KEY \ --out ./out.md ``` @@ -123,7 +91,7 @@ Output behavior: - Artifact content goes to stdout unless `--out` is set. - Summaries and errors are written to stderr. -- Exit code `2` indicates run succeeded but validation status is `failed`. +- Exit code `2` means the run succeeded but validation status is `failed`. ### `scriptorium serve` @@ -138,23 +106,24 @@ Common optional flags: - `--addr` (default `:8080`) - `--schema-dir` (default `.`) -- `--llm-api-key` - `--model` - `--timeout` (default `10m`) ## HTTP API -Run endpoint: +Endpoint: - `POST /v1/runs` -- No built-in authentication is provided in the current implementation; deploy behind a trusted boundary or gateway. + +No built-in authentication is provided by the server itself. Deploy behind a trusted boundary or gateway. Request example: ```json { - "profile_id": "generic.structured_events", - "profile_version": "1.0.0", + "prompt_id": "generic.structured_events", + "prompt_version": "1.0.0", + "profile_id": "local-default", "inputs": { "transcript": {"type": "file", "uri": "./examples/fixtures/transcript.md"}, "glossary": {"type": "file", "uri": "./examples/fixtures/glossary.yml"} @@ -168,7 +137,8 @@ Request example: "temperature": 0.0, "max_tokens": 600, "top_p": 1.0, - "timeout_seconds": 120 + "timeout_seconds": 120, + "api_key_env": "SCRIPTORIUM_API_KEY" } } ``` @@ -195,9 +165,11 @@ Response shape: }, "metadata": { "run_id": "xxxxxxxx-xxxx-4xxx-8xxx-xxxxxxxxxxxx", - "profile_id": "generic.structured_events", - "profile_version": "1.0.0", - "profile_hash": "...", + "prompt_id": "generic.structured_events", + "prompt_version": "1.0.0", + "prompt_hash": "...", + "rendered_prompt_hash": "...", + "selected_profile_id": "local-default", "model_name": "gpt-4o-mini", "endpoint": "http://localhost:8000/v1", "model_params": { @@ -206,10 +178,10 @@ Response shape: "temperature": 0, "max_tokens": 600, "top_p": 1, - "timeout_seconds": 120 + "timeout_seconds": 120, + "api_key_env": "SCRIPTORIUM_API_KEY" }, "input_hashes": {"transcript": "...", "glossary": "..."}, - "prompt_hash": "...", "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, "start_time": "...", "end_time": "...", @@ -222,7 +194,7 @@ Response shape: } ``` -Validation content failures are returned as successful run responses (`200`) with `validation.status = "failed"`; raw model output is preserved in `raw_model_output`. +Validation content failures return `200` with `validation.status = "failed"` and preserve `raw_model_output`. Error response shape: @@ -235,15 +207,17 @@ Error response shape: } ``` -## Prompt Profile Authoring +## Prompt Definition Authoring -### Minimal Markdown profile +### Minimal Markdown prompt definition ```yaml id: generic.markdown_summary version: "1.0.0" -expected_inputs: - - transcript +default_profile: local-default +inputs: + - name: transcript + required: true templates: - role: system content: "You are a concise assistant." @@ -251,23 +225,20 @@ templates: content: | Summarize: {{input "transcript"}} -model_defaults: - endpoint: http://localhost:8000/v1 - model: gpt-4o-mini - temperature: 0.2 - max_tokens: 700 output_format: markdown validation: validation_mode: basic ``` -### Structured JSON profile with schema validation +### Structured JSON prompt definition with schema validation ```yaml id: generic.structured_events version: "1.0.0" -expected_inputs: - - transcript +default_profile: local-default +inputs: + - name: transcript + required: true templates: - role: system content: "Return only JSON." @@ -275,9 +246,6 @@ templates: content: | Extract events from: {{input "transcript"}} -model_defaults: - endpoint: http://localhost:8000/v1 - model: gpt-4o-mini output_format: json validation: format: json @@ -286,30 +254,28 @@ validation: repair_attempts: 1 ``` -`repair_attempts` is bounded. Repair is attempted only for structured validation modes. +`repair_attempts` is strictly bounded and only applies to structured validation modes. ## Validation Modes -Supported modes: - - `none`: skipped validation result. -- `basic`: fails if output is empty/whitespace. +- `basic`: fails for empty/whitespace output. - `json`: output must parse as JSON. -- `json_schema`: output must parse as JSON and satisfy the configured schema. +- `json_schema`: output must parse as JSON and satisfy configured schema. -Validation failures caused by output content are represented in `validation` and do not discard raw model output. +Validation content failures are returned in the structured result; raw model output is preserved. -## Repository Examples +## Examples -- Profiles: `profiles/` +- Prompt definitions: `profiles/` - Schemas: `schemas/` - Fixtures: `examples/fixtures/` - Local experimentation: `local-test/` ## Development Notes -- Core is generic and follows a ports-and-adapters style. -- Domain/usecase packages do not depend on HTTP/CLI/wire types. +- Core follows ports-and-adapters and remains domain-generic. +- Domain/usecase packages do not depend on HTTP/CLI wire DTOs. - To add a new LLM adapter: implement `internal/llm.Client`. -- To add a new artifact reader: implement/extend `internal/artifact.Reader` routing. -- To add a new validation mode: extend `internal/validate` and keep run semantics stable. +- To add a new artifact reader: extend `internal/artifact.Reader` routing. +- To add a new validation mode: extend `internal/validate` and preserve run semantics. diff --git a/architecture.md b/architecture.md index b7e2e0c..d2249be 100644 --- a/architecture.md +++ b/architecture.md @@ -2,11 +2,11 @@ ## 1. Purpose and Non-Goals -Scriptorium is a prompt-profile execution engine. +Scriptorium is a prompt-definition execution engine. -It takes named input artifacts, renders prompt templates, calls an LLM, validates output, optionally performs bounded structured-output repair, and returns an artifact with metadata. +It accepts named input artifacts, renders prompt templates, calls an LLM, validates output, optionally performs bounded structured-output repair, and returns an artifact with metadata. -Scriptorium is not an orchestrator. It should not own transcription, transcript merging, transcript polishing, notification, or cross-step workflow control. +Scriptorium is not an orchestrator. It must not own transcription, transcript merge/polish steps, notifications, or cross-step workflow control. For the motivating D&D workflow: @@ -14,111 +14,116 @@ For the motivating D&D workflow: - WhisperX transcribes. - Seriatim merges transcripts. - Audita polishes transcripts. -- Scriptorium generates output artifacts from prepared inputs. +- Scriptorium generates final artifacts from prepared inputs. -Core Go code must remain domain-generic. +Core Go code remains generic. ## 2. Current Architecture -Current high-level structure: +Current implementation structure: - `cmd/scriptorium`: binary entrypoint. -- `internal/domain`: core domain types. -- `internal/usecase`: `Runner` use case and repair loop orchestration. -- `internal/profile`: filesystem prompt profile repository and profile validation. -- `internal/artifact`: artifact reference readers (`inline`, `file`) and routing. -- `internal/prompt`: Go template-based prompt renderer. -- `internal/llm`: LLM client interface + OpenAI-compatible HTTP adapter. -- `internal/validate`: output validator implementation (`none/basic/json/json_schema`). +- `internal/domain`: core domain contracts. +- `internal/usecase`: `Runner` run flow, validation integration, bounded repair coordination. +- `internal/profile`: transitional filesystem prompt-definition repository (package rename deferred). +- `internal/artifact`: input artifact resolution (`inline`, `file`). +- `internal/prompt`: template rendering. +- `internal/llm`: provider-neutral client interface + OpenAI-compatible HTTP adapter. +- `internal/validate`: validation implementation (`none/basic/json/json_schema`). - `internal/adapter/cli`: CLI adapter. - `internal/adapter/http`: HTTP adapter (`POST /v1/runs`). -This is a practical ports-and-adapters implementation. - ## 3. Run Data Flow `Runner.Run(ctx, RunRequest)` currently executes: -1. Validate minimum request requirements (`profile_id`). -2. Load prompt profile by ID/version. -3. Merge effective model target (profile defaults + request override). -4. Resolve effective output contract (profile + optional request override). -5. Resolve named artifact refs to loaded artifacts. -6. Render prompt messages from templates. -7. Hash rendered prompt for auditability. -8. Call LLM client with provider-neutral `GenerateRequest`. -9. Build output artifact from model content. +1. Validate request (`prompt_id` required). +2. Load `PromptDefinition` by ID/version. +3. Determine selected profile ID (`request.profile_id` or prompt `default_profile`). +4. Resolve effective execution target from request override (execution-profile loading is deferred in this phase). +5. Resolve named input artifact refs. +6. Render prompt messages. +7. Hash prompt definition and rendered prompt. +8. Call LLM client with `GenerateRequest`. +9. Build output artifact. 10. Validate output. -11. If structured validation failed and repair is enabled/bounded, run repair attempts and re-validate. -12. Return `RunResult` with artifact, validation, raw output, and metadata. +11. If structured validation failed and repair is enabled, run bounded repair attempts and re-validate. +12. Return `RunResult` with artifact, raw output, validation result, metadata. -Validation content failure remains a successful run result with `validation.status=failed`. +Validation content failures are returned as successful runs with `validation.status=failed`. ## 4. Package Responsibilities - `domain` - - Owns core nouns and contracts. - - Must not import adapters/provider SDK types. + - Owns core nouns/contracts. + - Must not depend on adapters/provider SDK types. - `usecase` - - Owns execution sequence and cross-port orchestration for a single run. - - May coordinate validation and bounded repair. - - Must not contain HTTP/CLI/wire concerns. + - Owns single-run orchestration across ports. + - Owns bounded repair control flow. + - Must not own transport/wire concerns. -- `profile` - - Owns prompt profile loading/parsing/validation. - - Handles YAML strict decoding and profile-level constraints. +- `profile` (transitional) + - Currently loads prompt definitions from YAML. + - Package naming split (`prompt definition repo` vs `execution profile repo`) is deferred follow-up. - `artifact` - - Owns artifact ref resolution and content loading. - - Produces normalized `Artifact` values with size/hash/content type. + - Loads artifacts from refs and normalizes payload metadata. - `prompt` - - Owns template rendering and required-input enforcement. + - Renders templates and enforces required inputs. - `llm` - - Owns generation port and provider adapters. - - Current adapter: OpenAI-compatible chat completions over `net/http`. + - Defines generation client contract and protocol adapters. - `validate` - - Owns output validation semantics and JSON Schema integration. + - Owns output validation semantics and schema validation. - `adapter/http`, `adapter/cli` - - Owns transport/wire/flag concerns only. - - Should stay thin and delegate business flow to `usecase.Runner`. + - Own request/response/flag mapping only. + - Delegate business flow to `usecase.Runner`. ## 5. Domain Model (Current) -Key types in `internal/domain`: +Key types: -- `RunRequest`: profile selector, named input refs, vars, optional model override, optional validation override. -- `RunResult`: output artifact, validation result, raw output, profile/model metadata, hashes, usage, timestamps, duration. -- `ArtifactRef`: `{type, uri, body}` reference contract. -- `Artifact`: loaded payload (`name`, `content_type`, `body`, `uri`, `size`, `hash`). -- `PromptProfile`: YAML-backed profile definition. -- `RenderedPrompt` / `RenderedMessage`: provider-neutral prompt structure. -- `GenerateRequest` / `GenerateResponse`: provider-neutral model I/O. -- `ValidationResult`: passed/failed/skipped + mode/errors/schema/repair attempts. +- `PromptDefinition` + - `id`, `version`, `default_profile`, `inputs`, `templates`, `output_format`, `validation`. +- `ExecutionProfile` + - Execution/runtime settings shape (`endpoint`, `model`, timeouts, `api_key_env`, etc.). + - Loading/persistence is deferred in this pass. +- `ExecutionTarget` + - Effective execution settings for a run. +- `RunRequest` + - `prompt_id`, `prompt_version`, optional `profile_id`, `inputs`, `vars`, optional `execution` override, optional validation override. +- `RunResult` + - Output artifact, validation, raw output, prompt/profile/model metadata, hashes, timing, usage. +- `ArtifactRef` / `Artifact` + - Input reference and loaded content contracts. +- `RenderedPrompt` / `RenderedMessage` + - Provider-neutral rendered prompt. +- `GenerateRequest` / `GenerateResponse` + - Provider-neutral model I/O. ## 6. Interfaces and Adapters Primary ports: -- `profile.Repository` +- `profile.Repository` (transitional prompt-definition lookup) - `artifact.Reader` - `prompt.Renderer` - `llm.Client` - `validate.Validator` -- `usecase.OutputRepairer` (usecase-local abstraction) +- `usecase.OutputRepairer` (usecase-local) Current adapters: -- Profile repository: filesystem YAML loader. -- Artifact reader: composite reader for `inline` and `file`. -- Prompt renderer: Go templates with input helper + vars. -- LLM adapter: OpenAI-compatible `/chat/completions`. -- Validator: standard validator with `none/basic/json/json_schema`. -- CLI/HTTP adapters: thin request mapping and response mapping. +- Prompt definition repository: filesystem YAML loader. +- Artifact readers: `file`, `inline` via composite reader. +- Prompt renderer: Go templates with `input` helper. +- LLM adapter: OpenAI-compatible `/chat/completions` over `net/http`. +- Validator: standard validator (`none/basic/json/json_schema`). +- CLI/HTTP adapters. ## 7. Validation and Repair Model @@ -131,12 +136,11 @@ Validation modes: Repair behavior: -- Only applies to structured modes (`json`, `json_schema`). -- Attempted only when validation fails, repairer exists, and `repair_attempts > 0`. -- Bounded strictly by `repair_attempts`. -- Uses a narrow JSON-repair prompt and re-validates each attempt. -- If still invalid, run succeeds with failed validation and preserved final raw output. -- Validator runtime/config errors are run errors. +- Applies only to structured modes (`json`, `json_schema`). +- Triggered only on failed validation and only when `repair_attempts > 0`. +- Strictly bounded by `repair_attempts`. +- Uses a narrow repair prompt asking for corrected JSON only. +- Runtime validator/repair errors are run errors. ## 8. Public Contracts @@ -147,86 +151,76 @@ Commands: - `scriptorium run` - `scriptorium serve` -`run`: +`run` flags: -- Required: `--profile-dir`, `--profile-id`, `--input`. -- Optional: model/endpoint overrides (`--model`, `--llm-base-url`), vars, output path, schema dir, timeout. -- Artifact bytes go to stdout (or `--out` file); summaries/errors go to stderr. +- Required: `--profile-dir`, `--prompt-id`, `--input`. +- Optional: `--profile-id`, `--var`, `--out`, `--llm-base-url`, `--model`, `--api-key-env`, `--temperature`, `--max-tokens`, `--schema-dir`, `--timeout`. -`serve`: +Current transitional runtime behavior: -- Required: `--profile-dir`, `--llm-base-url`. -- Exposes HTTP run endpoint. +- Prompt definitions may provide `default_profile` selection. +- Execution-profile loading is deferred; execution settings must currently be supplied via run-time overrides. ### HTTP - Endpoint: `POST /v1/runs`. -- Request maps to `RunRequest` (`profile_id`, `inputs`, `vars`, optional `model` override). +- Request maps to `RunRequest` with `prompt_id` (required), `inputs`, optional `profile_id`, `vars`, optional execution override (`model` object). - Response includes `artifact`, `validation`, `metadata`, `raw_model_output`. -- Validation content failures are represented as `200` with `validation.status=failed`. -- Error responses are `{error:{code,message}}` with stable code mapping. +- Validation content failures return `200` with failed validation status. +- Error response shape: `{ "error": { "code": "...", "message": "..." } }`. -### Prompt Profile YAML +### Prompt Definition YAML -- `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation`. -- Strict YAML decoding (`KnownFields`) rejects unknown fields. -- `validation.schema_path` required when `validation_mode=json_schema`. -- `validation.repair_attempts` must be non-negative. +Current prompt-definition fields: -### Metadata +- `id`, `version`, optional `default_profile`, optional `description` +- `inputs[]` with `name`, `required`, optional `content_type`, optional `description` +- `templates[]` with `role` and either `content` or `content_file` +- `output_format` +- `validation` (`format`, `validation_mode`, `schema_path`, `repair_attempts`) -Current run metadata includes: +Strict YAML decoding (`KnownFields`) is enabled. -- `run_id` (UUID v4) -- `profile_id`, `profile_version`, `profile_hash` -- `model_name`, `endpoint`, effective `model_params` -- `input_hashes`, `prompt_hash` -- token usage -- start/end timestamps -- duration -- validation mode/status -- repair attempts used +### API Key Policy + +- Raw API keys are not accepted in YAML, CLI flags, HTTP body, or domain metadata. +- Auth is configured only by env var reference (`api_key_env`), resolved at request time by the LLM adapter. ## 9. Extension Points (Future Work) -Future features should plug into existing boundaries, not bypass them. +Planned next extensions should reuse current boundaries: -Candidate extensions: - -- S3 artifact refs via `artifact.Reader` extension. -- Token budgeting in usecase/model-target policy layer. -- Streaming LLM output via additional `llm.Client` methods/adapters. -- Batch execution as a separate use case (not hidden in single-run path). -- Additional LLM providers implementing `llm.Client`. -- Additional validators/modes in `validate`. -- Additional profile repositories (embedded, remote, object storage). - -These are future work, not part of current default behavior. +- Execution-profile repository/loader implementation. +- Split transitional `internal/profile` into clearer prompt-definition/profile repositories. +- S3 artifact refs. +- Token budgeting/policy layer. +- Streaming generation. +- Batch run use case. +- Additional provider adapters. +- Additional validation modes. ## 10. Architectural Guardrails -Contributors should preserve these constraints: - -- No D&D-specific behavior in core Go packages. +- No D&D-specific logic in core Go packages. - No orchestration creep into Scriptorium. - No unbounded repair loops. -- No silent truncation/omission of rendered inputs or outputs. -- Do not log full artifacts/prompts by default. +- No silent content truncation/omission. +- Do not log full prompts/artifacts by default. - Keep provider-specific wire/SDK details out of domain types. -- Keep adapter boundaries explicit and thin. +- Keep adapters thin. ## 11. Testing Strategy -Protect behavior at boundaries and in usecase flow: +Protect these behaviors with focused tests: -- Profile loading/parsing/validation errors. -- Artifact reading for inline/file + hash/content type behavior. -- Prompt rendering required inputs/template error behavior. -- LLM adapter request/response/error/timeout behavior. -- Runner success path and metadata population. +- Prompt-definition loading/validation errors. +- Artifact loading/hash/content-type behavior. +- Prompt rendering required-input and template error paths. +- LLM adapter request/response/auth/error/timeout behavior. +- Runner success/failure/metadata behavior. - Validation failure raw-output preservation. -- Successful/failed/bounded repair flows. -- HTTP request mapping, response shape, and error mapping. -- CLI parsing helpers, required flags, and output stream separation. +- Bounded repair behavior. +- HTTP mapping and error mapping. +- CLI parsing and output stream separation. -Prefer focused unit tests and small integration-style tests with fake LLMs. +Prefer small unit tests and minimal integration-style tests with fake LLMs. diff --git a/internal/adapter/cli/run.go b/internal/adapter/cli/run.go index eef43dd..fcbf963 100644 --- a/internal/adapter/cli/run.go +++ b/internal/adapter/cli/run.go @@ -30,12 +30,13 @@ const ( type runConfig struct { profileDir string + promptID string profileID string inputRaw listFlag varRaw listFlag outputPath string llmBaseURL string - llmAPIKey string + apiKeyEnv string model string temperature float64 maxTokens int @@ -43,6 +44,7 @@ type runConfig struct { timeout time.Duration llmBaseURLSet bool + apiKeyEnvSet bool modelSet bool temperatureSet bool maxTokensSet bool @@ -53,7 +55,6 @@ type serveConfig struct { profileDir string schemaDir string llmBaseURL string - llmAPIKey string model string timeout time.Duration } @@ -115,7 +116,6 @@ func runCommand(args []string, stdout, stderr io.Writer) int { llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ BaseURL: cfg.llmBaseURL, - APIKey: cfg.llmAPIKey, Model: cfg.model, Timeout: cfg.timeout, }) @@ -132,21 +132,24 @@ func runCommand(args []string, stdout, stderr io.Writer) int { validate.NewStandardValidator(cfg.schemaDir), ) - var modelOverride *domain.ModelTarget - if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet { - modelOverride = &domain.ModelTarget{ - Endpoint: cfg.llmBaseURL, - Model: cfg.model, - Temperature: cfg.temperature, - MaxTokens: cfg.maxTokens, + var modelOverride *domain.ExecutionTarget + if cfg.llmBaseURLSet || cfg.modelSet || cfg.temperatureSet || cfg.maxTokensSet || cfg.apiKeyEnvSet { + modelOverride = &domain.ExecutionTarget{ + Endpoint: cfg.llmBaseURL, + Model: cfg.model, + Temperature: cfg.temperature, + MaxTokens: cfg.maxTokens, + TimeoutSeconds: int(cfg.timeout.Seconds()), + APIKeyEnv: cfg.apiKeyEnv, } } res, runErr := runner.Run(context.Background(), domain.RunRequest{ + PromptID: cfg.promptID, ProfileID: cfg.profileID, Inputs: inputs, Vars: varMappings, - Model: modelOverride, + Execution: modelOverride, }) if runErr != nil { fmt.Fprintf(stderr, "run error: %v\n", runErr) @@ -171,7 +174,6 @@ func serveCommand(args []string, stderr io.Writer) int { llmClient, err := llm.NewOpenAICompatibleClient(llm.OpenAICompatibleConfig{ BaseURL: cfg.llmBaseURL, - APIKey: cfg.llmAPIKey, Model: cfg.model, Timeout: cfg.timeout, }) @@ -208,13 +210,14 @@ func parseRunArgs(args []string) (*runConfig, error) { fs := flag.NewFlagSet("run", flag.ContinueOnError) fs.SetOutput(io.Discard) - fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt profile YAML files") - fs.StringVar(&cfg.profileID, "profile-id", "", "profile ID to run") + fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt definition YAML files") + fs.StringVar(&cfg.promptID, "prompt-id", "", "prompt ID to run") + fs.StringVar(&cfg.profileID, "profile-id", "", "optional execution profile ID; if omitted, prompt default_profile is used") fs.Var(&cfg.inputRaw, "input", "input mapping(s): name=path (repeatable, comma-separated)") fs.Var(&cfg.varRaw, "var", "variable mapping(s): name=value (repeatable, comma-separated)") fs.StringVar(&cfg.outputPath, "out", "", "optional output file path") fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1") - fs.StringVar(&cfg.llmAPIKey, "llm-api-key", "", "optional API key") + fs.StringVar(&cfg.apiKeyEnv, "api-key-env", "", "environment variable name containing API key") fs.StringVar(&cfg.model, "model", "", "model name") fs.Float64Var(&cfg.temperature, "temperature", 0, "optional temperature override") fs.IntVar(&cfg.maxTokens, "max-tokens", 0, "optional max tokens override") @@ -231,8 +234,8 @@ func parseRunArgs(args []string) (*runConfig, error) { if strings.TrimSpace(cfg.profileDir) == "" { return nil, errors.New("--profile-dir is required") } - if strings.TrimSpace(cfg.profileID) == "" { - return nil, errors.New("--profile-id is required") + if strings.TrimSpace(cfg.promptID) == "" { + return nil, errors.New("--prompt-id is required") } if len(cfg.inputRaw) == 0 { return nil, errors.New("at least one --input is required") @@ -243,6 +246,7 @@ func parseRunArgs(args []string) (*runConfig, error) { cfg.outputPath = filepath.Clean(cfg.outputPath) } cfg.llmBaseURLSet = flagWasSet(fs, "llm-base-url") + cfg.apiKeyEnvSet = flagWasSet(fs, "api-key-env") cfg.modelSet = flagWasSet(fs, "model") cfg.temperatureSet = flagWasSet(fs, "temperature") cfg.maxTokensSet = flagWasSet(fs, "max-tokens") @@ -256,10 +260,9 @@ func parseServeArgs(args []string) (*serveConfig, error) { fs.SetOutput(io.Discard) fs.StringVar(&cfg.addr, "addr", ":8080", "HTTP listen address") - fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt profile YAML files") + fs.StringVar(&cfg.profileDir, "profile-dir", "", "directory containing prompt definition YAML files") fs.StringVar(&cfg.schemaDir, "schema-dir", ".", "base directory for validation schemas") fs.StringVar(&cfg.llmBaseURL, "llm-base-url", "", "OpenAI-compatible base URL including /v1") - fs.StringVar(&cfg.llmAPIKey, "llm-api-key", "", "optional API key") fs.StringVar(&cfg.model, "model", "", "optional default model") fs.DurationVar(&cfg.timeout, "timeout", 10*time.Minute, "LLM request timeout") @@ -351,14 +354,15 @@ func printSummary(stderr io.Writer, res *domain.RunResult) { if res == nil { return } - fmt.Fprintf(stderr, "profile=%s@%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n", - res.ProfileID, - res.ProfileVersion, + fmt.Fprintf(stderr, "prompt=%s@%s selected_profile=%s model=%s validation=%s mode=%s validation_errors=%d prompt_hash=%s inputs=%d usage=%d/%d/%d\n", + res.PromptID, + res.PromptVersion, + res.SelectedProfileID, res.ModelName, res.Validation.Status, res.Validation.Mode, len(res.Validation.Errors), - res.PromptHash, + res.RenderedPromptHash, len(res.InputHashes), res.Usage.PromptTokens, res.Usage.CompletionTokens, @@ -368,6 +372,6 @@ func printSummary(stderr io.Writer, res *domain.RunResult) { func printUsage(w io.Writer) { fmt.Fprintln(w, "usage: scriptorium ...") - fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --profile-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--var k=v] [--out path] [--timeout 10m]") - fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--llm-api-key KEY] [--model NAME] [--timeout 10m]") + fmt.Fprintln(w, " run: scriptorium run --profile-dir DIR --prompt-id ID --input name=path [--input ...] [--llm-base-url URL] [--model NAME] [--api-key-env ENV] [--var k=v] [--out path] [--timeout 10m]") + fmt.Fprintln(w, " serve: scriptorium serve --addr :8080 --profile-dir DIR --llm-base-url URL [--schema-dir DIR] [--model NAME] [--timeout 10m]") } diff --git a/internal/adapter/cli/run_test.go b/internal/adapter/cli/run_test.go index e181ddb..6e8e528 100644 --- a/internal/adapter/cli/run_test.go +++ b/internal/adapter/cli/run_test.go @@ -51,17 +51,17 @@ func TestParseMappingsMalformed(t *testing.T) { } func TestParseRunArgsRequiredFlags(t *testing.T) { - _, err := parseRunArgs([]string{"--profile-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"}) + _, err := parseRunArgs([]string{"--prompt-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"}) if err == nil { t.Fatal("expected missing --profile-dir error") } _, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m"}) if err == nil { - t.Fatal("expected missing --profile-id error") + t.Fatal("expected missing --prompt-id error") } - _, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--profile-id", "p", "--llm-base-url", "http://x/v1", "--model", "m"}) + _, err = parseRunArgs([]string{"--profile-dir", "./profiles", "--prompt-id", "p", "--llm-base-url", "http://x/v1", "--model", "m"}) if err == nil { t.Fatal("expected missing --input error") } @@ -70,7 +70,7 @@ func TestParseRunArgsRequiredFlags(t *testing.T) { func TestParseRunArgsAllowsOmittedModelAndBaseURL(t *testing.T) { cfg, err := parseRunArgs([]string{ "--profile-dir", "./profiles", - "--profile-id", "p", + "--prompt-id", "p", "--input", "a=b", }) if err != nil { @@ -107,7 +107,7 @@ func TestParseServeArgsRequiredFlags(t *testing.T) { func TestParseRunArgsTimeout(t *testing.T) { cfg, err := parseRunArgs([]string{ "--profile-dir", "./profiles", - "--profile-id", "p", + "--prompt-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m", @@ -121,7 +121,7 @@ func TestParseRunArgsTimeout(t *testing.T) { cfg, err = parseRunArgs([]string{ "--profile-dir", "./profiles", - "--profile-id", "p", + "--prompt-id", "p", "--input", "a=b", "--llm-base-url", "http://x/v1", "--model", "m", @@ -156,7 +156,7 @@ func TestRunCommandVarsOptional(t *testing.T) { code := runCommand([]string{ "--profile-dir", "./profiles", - "--profile-id", "p", + "--prompt-id", "p", "--input", "transcript=./t.md", "--llm-base-url", "://bad-url", "--model", "m", @@ -184,18 +184,19 @@ func TestWriteOutputAndSummaryUseSeparateWriters(t *testing.T) { t.Fatalf("unexpected writeOutput error: %v", err) } printSummary(&stderr, &domain.RunResult{ - ProfileID: "p", - ProfileVersion: "1", - ModelName: "m", - Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic}, - PromptHash: "h", - InputHashes: map[string]string{"in": "x"}, + PromptID: "p", + PromptVersion: "1", + SelectedProfileID: "exec", + ModelName: "m", + Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic}, + RenderedPromptHash: "h", + InputHashes: map[string]string{"in": "x"}, }) if stdout.String() != "artifact-body" { t.Fatalf("expected artifact output on stdout, got %q", stdout.String()) } - if !strings.Contains(stderr.String(), "profile=p@1") { + if !strings.Contains(stderr.String(), "prompt=p@1") { t.Fatalf("expected summary on stderr, got %q", stderr.String()) } } diff --git a/internal/adapter/http/dto.go b/internal/adapter/http/dto.go index a681fe4..3d91e7d 100644 --- a/internal/adapter/http/dto.go +++ b/internal/adapter/http/dto.go @@ -5,11 +5,12 @@ import ( ) type runRequestDTO struct { - ProfileID string `json:"profile_id"` - ProfileVersion string `json:"profile_version,omitempty"` - Inputs map[string]inputRefDTO `json:"inputs"` - Vars map[string]string `json:"vars,omitempty"` - Model *modelOverrideRequestDTO `json:"model,omitempty"` + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + Inputs map[string]inputRefDTO `json:"inputs"` + Vars map[string]string `json:"vars,omitempty"` + Model *modelOverrideRequestDTO `json:"model,omitempty"` } type inputRefDTO struct { @@ -19,12 +20,15 @@ type inputRefDTO struct { } type modelOverrideRequestDTO struct { - Endpoint string `json:"endpoint,omitempty"` - Model string `json:"model,omitempty"` - Temperature float64 `json:"temperature,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopP float64 `json:"top_p,omitempty"` - TimeoutSeconds int `json:"timeout_seconds,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Model string `json:"model,omitempty"` + Temperature float64 `json:"temperature,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + TopP float64 `json:"top_p,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` + APIKeyEnv string `json:"api_key_env,omitempty"` + ExtraParams map[string]string `json:"extra_params,omitempty"` } type runResponseDTO struct { @@ -45,14 +49,15 @@ type artifactDTO struct { type metadataDTO struct { RunID string `json:"run_id"` - ProfileID string `json:"profile_id"` - ProfileVersion string `json:"profile_version"` - ProfileHash string `json:"profile_hash"` + PromptID string `json:"prompt_id"` + PromptVersion string `json:"prompt_version"` + PromptHash string `json:"prompt_hash"` + RenderedPromptHash string `json:"rendered_prompt_hash"` + SelectedProfileID string `json:"selected_profile_id"` ModelName string `json:"model_name"` Endpoint string `json:"endpoint"` ModelParams modelParamsDTO `json:"model_params"` InputHashes map[string]string `json:"input_hashes"` - PromptHash string `json:"prompt_hash"` Usage tokenUsageDTO `json:"usage"` StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` @@ -63,12 +68,15 @@ type metadataDTO struct { } type modelParamsDTO struct { - Endpoint string `json:"endpoint"` - Model string `json:"model"` - Temperature float64 `json:"temperature"` - MaxTokens int `json:"max_tokens"` - TopP float64 `json:"top_p"` - TimeoutSeconds int `json:"timeout_seconds"` + Endpoint string `json:"endpoint"` + Model string `json:"model"` + Temperature float64 `json:"temperature"` + MaxTokens int `json:"max_tokens"` + TopP float64 `json:"top_p"` + TimeoutSeconds int `json:"timeout_seconds"` + ReasoningEffort string `json:"reasoning_effort,omitempty"` + APIKeyEnv string `json:"api_key_env,omitempty"` + ExtraParams map[string]string `json:"extra_params,omitempty"` } type tokenUsageDTO struct { diff --git a/internal/adapter/http/handler.go b/internal/adapter/http/handler.go index 8e3a23e..cad1748 100644 --- a/internal/adapter/http/handler.go +++ b/internal/adapter/http/handler.go @@ -40,8 +40,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if strings.TrimSpace(req.ProfileID) == "" { - writeError(w, http.StatusBadRequest, "invalid_request", "profile_id is required") + if strings.TrimSpace(req.PromptID) == "" { + writeError(w, http.StatusBadRequest, "invalid_request", "prompt_id is required") return } if len(req.Inputs) == 0 { @@ -58,24 +58,28 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } - var model *domain.ModelTarget + var model *domain.ExecutionTarget if req.Model != nil { - model = &domain.ModelTarget{ - Endpoint: req.Model.Endpoint, - Model: req.Model.Model, - Temperature: req.Model.Temperature, - MaxTokens: req.Model.MaxTokens, - TopP: req.Model.TopP, - TimeoutSeconds: req.Model.TimeoutSeconds, + model = &domain.ExecutionTarget{ + Endpoint: req.Model.Endpoint, + Model: req.Model.Model, + Temperature: req.Model.Temperature, + MaxTokens: req.Model.MaxTokens, + TopP: req.Model.TopP, + TimeoutSeconds: req.Model.TimeoutSeconds, + ReasoningEffort: req.Model.ReasoningEffort, + APIKeyEnv: req.Model.APIKeyEnv, + ExtraParams: req.Model.ExtraParams, } } res, err := h.runner.Run(r.Context(), domain.RunRequest{ - ProfileID: req.ProfileID, - ProfileVersion: req.ProfileVersion, - Inputs: mappedInputs, - Vars: req.Vars, - Model: model, + PromptID: req.PromptID, + PromptVersion: req.PromptVersion, + ProfileID: req.ProfileID, + Inputs: mappedInputs, + Vars: req.Vars, + Execution: model, }) if err != nil { status, code, message := mapRunError(err) @@ -94,22 +98,26 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { }, Validation: mapValidation(res.Validation), Metadata: metadataDTO{ - RunID: res.RunID, - ProfileID: res.ProfileID, - ProfileVersion: res.ProfileVersion, - ProfileHash: res.ProfileHash, - ModelName: res.ModelName, - Endpoint: res.Endpoint, + RunID: res.RunID, + PromptID: res.PromptID, + PromptVersion: res.PromptVersion, + PromptHash: res.PromptHash, + RenderedPromptHash: res.RenderedPromptHash, + SelectedProfileID: res.SelectedProfileID, + ModelName: res.ModelName, + Endpoint: res.Endpoint, ModelParams: modelParamsDTO{ - Endpoint: res.ModelParams.Endpoint, - Model: res.ModelParams.Model, - Temperature: res.ModelParams.Temperature, - MaxTokens: res.ModelParams.MaxTokens, - TopP: res.ModelParams.TopP, - TimeoutSeconds: res.ModelParams.TimeoutSeconds, + Endpoint: res.EffectiveModelParams.Endpoint, + Model: res.EffectiveModelParams.Model, + Temperature: res.EffectiveModelParams.Temperature, + MaxTokens: res.EffectiveModelParams.MaxTokens, + TopP: res.EffectiveModelParams.TopP, + TimeoutSeconds: res.EffectiveModelParams.TimeoutSeconds, + ReasoningEffort: res.EffectiveModelParams.ReasoningEffort, + APIKeyEnv: res.EffectiveModelParams.APIKeyEnv, + ExtraParams: res.EffectiveModelParams.ExtraParams, }, InputHashes: res.InputHashes, - PromptHash: res.PromptHash, Usage: tokenUsageDTO{ PromptTokens: res.Usage.PromptTokens, CompletionTokens: res.Usage.CompletionTokens, @@ -140,11 +148,11 @@ func mapValidation(v domain.ValidationResult) validationDTO { func mapRunError(err error) (int, string, string) { switch { case errors.Is(err, profile.ErrProfileNotFound): - return http.StatusNotFound, "profile_not_found", "profile not found" + return http.StatusNotFound, "prompt_not_found", "prompt definition not found" case errors.Is(err, usecase.ErrInvalidRequest): return http.StatusBadRequest, "invalid_request", "invalid run request" case errors.Is(err, usecase.ErrProfileLoad): - return http.StatusBadRequest, "profile_load_failed", "failed to load profile" + return http.StatusBadRequest, "prompt_load_failed", "failed to load prompt definition" case errors.Is(err, usecase.ErrArtifactLoad): return http.StatusBadRequest, "artifact_read_failed", "failed to read input artifact" case errors.Is(err, usecase.ErrPromptRender): diff --git a/internal/adapter/http/handler_test.go b/internal/adapter/http/handler_test.go index be608d4..6e80b42 100644 --- a/internal/adapter/http/handler_test.go +++ b/internal/adapter/http/handler_test.go @@ -42,13 +42,15 @@ func TestHandlerPostRunsSuccess(t *testing.T) { Size: 5, Hash: "abc", }, - Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true}, - ProfileID: "p1", - ProfileVersion: "1.0.0", - ProfileHash: "phash", - ModelName: "m1", - Endpoint: "http://llm/v1", - ModelParams: domain.ModelTarget{ + Validation: domain.ValidationResult{Status: domain.ValidationPassed, Mode: domain.ValidationBasic, IsValid: true}, + PromptID: "prompt-1", + PromptVersion: "1.0.0", + PromptHash: "phash", + RenderedPromptHash: "rhash", + SelectedProfileID: "exec-default", + ModelName: "m1", + Endpoint: "http://llm/v1", + EffectiveModelParams: domain.ExecutionTarget{ Endpoint: "http://llm/v1", Model: "m1", Temperature: 0.2, @@ -57,7 +59,6 @@ func TestHandlerPostRunsSuccess(t *testing.T) { TimeoutSeconds: 120, }, InputHashes: map[string]string{"transcript": "h1"}, - PromptHash: "ph", Usage: domain.TokenUsage{PromptTokens: 1, CompletionTokens: 2, TotalTokens: 3}, StartTime: start, EndTime: end, @@ -68,7 +69,8 @@ func TestHandlerPostRunsSuccess(t *testing.T) { h := NewHandler(r) body := []byte(`{ - "profile_id": "p1", + "prompt_id": "prompt-1", + "profile_id": "exec-default", "inputs": { "transcript": {"type": "file", "uri": "./t.md"} }, @@ -101,8 +103,8 @@ func TestHandlerPostRunsSuccess(t *testing.T) { if metadata["run_id"] != "11111111-1111-4111-8111-111111111111" { t.Fatalf("unexpected metadata.run_id: %#v", metadata["run_id"]) } - if metadata["profile_hash"] != "phash" { - t.Fatalf("unexpected metadata.profile_hash: %#v", metadata["profile_hash"]) + if metadata["prompt_hash"] != "phash" { + t.Fatalf("unexpected metadata.prompt_hash: %#v", metadata["prompt_hash"]) } usage := metadata["usage"].(map[string]any) if usage["total_tokens"] != float64(3) { @@ -122,14 +124,17 @@ func TestHandlerPostRunsSuccess(t *testing.T) { t.Fatalf("expected raw model output hello, got %#v", resp["raw_model_output"]) } - if r.last.ProfileID != "p1" { - t.Fatalf("expected request profile_id p1, got %q", r.last.ProfileID) + if r.last.PromptID != "prompt-1" { + t.Fatalf("expected request prompt_id prompt-1, got %q", r.last.PromptID) } - if r.last.Model == nil || r.last.Model.Model != "gpt-x" { - t.Fatalf("expected model override, got %#v", r.last.Model) + if r.last.ProfileID != "exec-default" { + t.Fatalf("expected request profile_id exec-default, got %q", r.last.ProfileID) } - if r.last.Model.TimeoutSeconds != 120 { - t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Model) + if r.last.Execution == nil || r.last.Execution.Model != "gpt-x" { + t.Fatalf("expected model override, got %#v", r.last.Execution) + } + if r.last.Execution.TimeoutSeconds != 120 { + t.Fatalf("expected timeout_seconds override 120, got %#v", r.last.Execution) } } @@ -145,7 +150,7 @@ func TestHandlerInvalidJSON(t *testing.T) { } } -func TestHandlerMissingProfileID(t *testing.T) { +func TestHandlerMissingPromptID(t *testing.T) { h := NewHandler(&fakeRunner{}) req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"inputs":{"x":{"type":"file","uri":"a"}}}`)) w := httptest.NewRecorder() @@ -173,7 +178,7 @@ func TestHandlerUsecaseErrorMapping(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { h := NewHandler(&fakeRunner{err: tc.err}) - req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"profile_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`)) + req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`)) w := httptest.NewRecorder() h.ServeHTTP(w, req) @@ -210,7 +215,7 @@ func TestHandlerValidationFailureStillSuccess(t *testing.T) { }, }}) - req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"profile_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`)) + req := httptest.NewRequest(http.MethodPost, "/v1/runs", bytes.NewBufferString(`{"prompt_id":"p","inputs":{"x":{"type":"file","uri":"a"}}}`)) w := httptest.NewRecorder() h.ServeHTTP(w, req) diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 1914c89..744c2c2 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -43,34 +43,36 @@ const ( // RunRequest represents a request to generate a single artifact. type RunRequest struct { - ProfileID string - ProfileVersion string - Inputs map[string]ArtifactRef - Vars map[string]string - Model *ModelTarget - Validation *OutputContract - Metadata map[string]string + PromptID string + PromptVersion string + ProfileID string + Inputs map[string]ArtifactRef + Vars map[string]string + Execution *ExecutionTarget + Validation *OutputContract + Metadata map[string]string } // RunResult represents the complete result of a prompt execution run. type RunResult struct { - RunID string - Artifact Artifact - RawOutput string - Validation ValidationResult - ProfileID string - ProfileVersion string - ProfileHash string - ModelName string - Endpoint string - ModelParams ModelTarget - InputHashes map[string]string - PromptHash string - Usage TokenUsage - StartTime time.Time - EndTime time.Time - Duration time.Duration - Error error + RunID string + Artifact Artifact + RawOutput string + Validation ValidationResult + PromptID string + PromptVersion string + PromptHash string + RenderedPromptHash string + SelectedProfileID string + ModelName string + Endpoint string + EffectiveModelParams ExecutionTarget + InputHashes map[string]string + Usage TokenUsage + StartTime time.Time + EndTime time.Time + Duration time.Duration + Error error } // ArtifactRef represents a reference to an input artifact. @@ -90,32 +92,58 @@ type Artifact struct { Hash string } -// PromptProfile represents a configured prompt execution profile. -type PromptProfile struct { +// PromptDefinition represents a configured prompt execution definition. +type PromptDefinition struct { ID string `yaml:"id"` Version string `yaml:"version"` + DefaultProfile string `yaml:"default_profile"` Description string `yaml:"description"` - ExpectedInputs []string `yaml:"expected_inputs"` + Inputs []PromptInput `yaml:"inputs"` Templates []PromptMessageTemplate `yaml:"templates"` - ModelDefaults ModelTarget `yaml:"model_defaults"` OutputFormat OutputFormat `yaml:"output_format"` Validation OutputContract `yaml:"validation"` } -// PromptMessageTemplate defines a template for a chat message. -type PromptMessageTemplate struct { - Role string `yaml:"role"` - Content string `yaml:"content"` +// PromptInput describes one named input expected by a prompt definition. +type PromptInput struct { + Name string `yaml:"name"` + Required bool `yaml:"required"` + ContentType string `yaml:"content_type"` + Description string `yaml:"description"` } -// ModelTarget represents the LLM endpoint and configuration. -type ModelTarget struct { - Endpoint string `yaml:"endpoint"` - Model string `yaml:"model"` - Temperature float64 `yaml:"temperature"` - MaxTokens int `yaml:"max_tokens"` - TopP float64 `yaml:"top_p"` - TimeoutSeconds int `yaml:"timeout_seconds"` +// PromptMessageTemplate defines a template for a chat message. +type PromptMessageTemplate struct { + Role string `yaml:"role"` + Content string `yaml:"content"` + ContentFile string `yaml:"content_file"` +} + +// ExecutionProfile describes how and where to execute a model. +type ExecutionProfile struct { + ID string `yaml:"id"` + Endpoint string `yaml:"endpoint"` + Model string `yaml:"model"` + Temperature float64 `yaml:"temperature"` + MaxTokens int `yaml:"max_tokens"` + TopP float64 `yaml:"top_p"` + TimeoutSeconds int `yaml:"timeout_seconds"` + ReasoningEffort string `yaml:"reasoning_effort"` + APIKeyEnv string `yaml:"api_key_env"` + ExtraParams map[string]string `yaml:"extra_params"` +} + +// ExecutionTarget represents effective model runtime settings for a run. +type ExecutionTarget struct { + Endpoint string `yaml:"endpoint" json:"endpoint"` + Model string `yaml:"model" json:"model"` + Temperature float64 `yaml:"temperature" json:"temperature"` + MaxTokens int `yaml:"max_tokens" json:"max_tokens"` + TopP float64 `yaml:"top_p" json:"top_p"` + TimeoutSeconds int `yaml:"timeout_seconds" json:"timeout_seconds"` + ReasoningEffort string `yaml:"reasoning_effort" json:"reasoning_effort"` + APIKeyEnv string `yaml:"api_key_env" json:"api_key_env"` + ExtraParams map[string]string `yaml:"extra_params" json:"extra_params"` } // OutputContract defines the requirements for the output artifact. @@ -140,7 +168,7 @@ type RenderedMessage struct { // GenerateRequest is the internal request passed to the LLM client. type GenerateRequest struct { Prompt RenderedPrompt - Target ModelTarget + Target ExecutionTarget } // GenerateResponse is the response received from the LLM client. @@ -168,19 +196,20 @@ type ValidationResult struct { // RunMetadata contains auditing information for a run. type RunMetadata struct { - RunID string - ProfileID string - ProfileVersion string - ProfileHash string - PromptHash string - InputHashes map[string]string - ModelEndpoint string - ModelName string - Params ModelTarget - Timestamp time.Time - Duration time.Duration - Usage TokenUsage - ValidationMode ValidationMode - ValidationStatus ValidationStatus - RepairAttempts int + RunID string + PromptID string + PromptVersion string + PromptHash string + RenderedPromptHash string + SelectedProfileID string + InputHashes map[string]string + ModelEndpoint string + ModelName string + Params ExecutionTarget + Timestamp time.Time + Duration time.Duration + Usage TokenUsage + ValidationMode ValidationMode + ValidationStatus ValidationStatus + RepairAttempts int } diff --git a/internal/llm/openai_compatible_client.go b/internal/llm/openai_compatible_client.go index 260be58..e02f257 100644 --- a/internal/llm/openai_compatible_client.go +++ b/internal/llm/openai_compatible_client.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "os" "net/http" "net/url" "strings" @@ -25,7 +26,6 @@ var ( type OpenAICompatibleConfig struct { BaseURL string - APIKey string Model string Timeout time.Duration HTTPClient *http.Client @@ -33,7 +33,6 @@ type OpenAICompatibleConfig struct { type OpenAICompatibleClient struct { baseURL string - apiKey string defaultModel string timeout time.Duration httpClient *http.Client @@ -64,7 +63,6 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli return &OpenAICompatibleClient{ baseURL: strings.TrimRight(baseURL, "/"), - apiKey: cfg.APIKey, defaultModel: cfg.Model, timeout: timeout, httpClient: client, @@ -125,8 +123,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err) } httpReq.Header.Set("Content-Type", "application/json") - if strings.TrimSpace(c.apiKey) != "" { - httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" { + apiKey := strings.TrimSpace(os.Getenv(envName)) + if apiKey == "" { + return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName) + } + httpReq.Header.Set("Authorization", "Bearer "+apiKey) } effectiveTimeout := c.timeout diff --git a/internal/llm/openai_compatible_client_test.go b/internal/llm/openai_compatible_client_test.go index 36e2182..bfadffd 100644 --- a/internal/llm/openai_compatible_client_test.go +++ b/internal/llm/openai_compatible_client_test.go @@ -44,23 +44,24 @@ func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ BaseURL: ts.URL + "/v1", - APIKey: "secret-key", Timeout: 2 * time.Second, }) if err != nil { t.Fatalf("unexpected constructor error: %v", err) } + t.Setenv("SCRIPTORIUM_TEST_API_KEY", "secret-key") resp, err := client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{ {Role: "system", Content: "You are helpful."}, {Role: "user", Content: "Say hello"}, }}, - Target: domain.ModelTarget{ + Target: domain.ExecutionTarget{ Model: "gpt-test", Temperature: 0.4, MaxTokens: 123, TopP: 0.7, + APIKeyEnv: "SCRIPTORIUM_TEST_API_KEY", }, }) if err != nil { @@ -110,7 +111,7 @@ func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{Model: "model"}, + Target: domain.ExecutionTarget{Model: "model"}, }) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -120,6 +121,29 @@ func TestOpenAICompatibleClientNoAuthorizationHeaderWhenNoAPIKey(t *testing.T) { } } +func TestOpenAICompatibleClientAPIKeyEnvMissing(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`)) + })) + defer ts.Close() + + client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{BaseURL: ts.URL + "/v1", Model: "model"}) + if err != nil { + t.Fatal(err) + } + + _, err = client.Generate(context.Background(), domain.GenerateRequest{ + Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, + Target: domain.ExecutionTarget{APIKeyEnv: "SCRIPTORIUM_MISSING_KEY"}, + }) + if err == nil { + t.Fatal("expected missing API key env error") + } + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } +} + func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) { gotModel := "" ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -139,7 +163,7 @@ func TestOpenAICompatibleClientModelFallbackFromConfig(t *testing.T) { _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{}, + Target: domain.ExecutionTarget{}, }) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -175,7 +199,7 @@ func TestOpenAICompatibleClientEndpointOverride(t *testing.T) { resp, err := client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{Endpoint: overrideServer.URL + "/v1"}, + Target: domain.ExecutionTarget{Endpoint: overrideServer.URL + "/v1"}, }) if err != nil { t.Fatalf("expected no error, got %v", err) @@ -306,7 +330,7 @@ func TestOpenAICompatibleClientRequestTimeoutOverride(t *testing.T) { resp, err := client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{TimeoutSeconds: 1}, + Target: domain.ExecutionTarget{TimeoutSeconds: 1}, }) if err != nil { t.Fatalf("expected request-level timeout override to succeed, got %v", err) @@ -327,7 +351,7 @@ func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) { _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{TimeoutSeconds: -1}, + Target: domain.ExecutionTarget{TimeoutSeconds: -1}, }) if err == nil { t.Fatal("expected invalid request error") @@ -348,7 +372,7 @@ func TestOpenAICompatibleClientAllowsEmptyConfiguredBaseURL(t *testing.T) { _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{Endpoint: "http://localhost:9999/v1"}, + Target: domain.ExecutionTarget{Endpoint: "http://localhost:9999/v1"}, }) if err == nil { t.Fatal("expected request failure due to unreachable endpoint") @@ -369,7 +393,7 @@ func TestOpenAICompatibleClientRequiresEndpointWhenUnsetEverywhere(t *testing.T) _, err = client.Generate(context.Background(), domain.GenerateRequest{ Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}}, - Target: domain.ModelTarget{}, + Target: domain.ExecutionTarget{}, }) if err == nil { t.Fatal("expected endpoint-required error") diff --git a/internal/profile/filesystem_repository.go b/internal/profile/filesystem_repository.go index 91dfefb..f98322f 100644 --- a/internal/profile/filesystem_repository.go +++ b/internal/profile/filesystem_repository.go @@ -13,9 +13,9 @@ import ( ) var ( - ErrProfileNotFound = errors.New("prompt profile not found") + ErrProfileNotFound = errors.New("prompt definition not found") ErrInvalidYAML = errors.New("invalid YAML format") - ErrInvalidProfile = errors.New("invalid profile configuration") + ErrInvalidProfile = errors.New("invalid prompt definition configuration") ) type filesystemRepository struct { @@ -26,9 +26,9 @@ func NewFilesystemRepository(dir string) Repository { return &filesystemRepository{dir: dir} } -func (r *filesystemRepository) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) { +func (r *filesystemRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { if strings.TrimSpace(id) == "" { - return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidProfile) } files, err := os.ReadDir(r.dir) @@ -53,7 +53,7 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string, versio return nil, fmt.Errorf("failed to read profile file %s: %w", file.Name(), err) } - var prof domain.PromptProfile + var prof domain.PromptDefinition decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) if err := decoder.Decode(&prof); err != nil { @@ -77,22 +77,33 @@ func (r *filesystemRepository) GetProfile(ctx context.Context, id string, versio return nil, ErrProfileNotFound } -func validateProfile(p *domain.PromptProfile) error { +func validateProfile(p *domain.PromptDefinition) error { if p.ID == "" { - return errors.New("profile id is required") + return errors.New("prompt id is required") } if p.Version == "" { - return errors.New("profile version is required") + return errors.New("prompt version is required") } if len(p.Templates) == 0 { return errors.New("at least one prompt template message is required") } + if len(p.Inputs) == 0 { + return errors.New("at least one prompt input is required") + } + for i, input := range p.Inputs { + if strings.TrimSpace(input.Name) == "" { + return fmt.Errorf("input %d has empty name", i) + } + } for i, t := range p.Templates { if !isValidMessageRole(t.Role) { return fmt.Errorf("template message %d has invalid role %q", i, t.Role) } - if t.Content == "" { - return fmt.Errorf("template message %d is missing content", i) + if strings.TrimSpace(t.Content) == "" && strings.TrimSpace(t.ContentFile) == "" { + return fmt.Errorf("template message %d must provide content or content_file", i) + } + if strings.TrimSpace(t.Content) != "" && strings.TrimSpace(t.ContentFile) != "" { + return fmt.Errorf("template message %d cannot set both content and content_file", i) } } if !isValidOutputFormat(p.OutputFormat) { @@ -107,17 +118,9 @@ func validateProfile(p *domain.PromptProfile) error { if p.Validation.ValidationMode == domain.ValidationJSONSchema && strings.TrimSpace(p.Validation.SchemaPath) == "" { return errors.New("validation.schema_path is required when validation_mode is json_schema") } - if p.ModelDefaults.TimeoutSeconds < 0 { - return errors.New("model_defaults.timeout_seconds must be greater than or equal to 0") - } if p.Validation.Format != "" && p.Validation.Format != p.OutputFormat { return fmt.Errorf("validation format %q does not match output format %q", p.Validation.Format, p.OutputFormat) } - for i, input := range p.ExpectedInputs { - if strings.TrimSpace(input) == "" { - return fmt.Errorf("expected input %d has empty name", i) - } - } return nil } diff --git a/internal/profile/repository.go b/internal/profile/repository.go index 2e13b5b..ae18605 100644 --- a/internal/profile/repository.go +++ b/internal/profile/repository.go @@ -5,7 +5,9 @@ import ( "gitea.maximumdirect.net/eric/scriptorium/internal/domain" ) -// Repository handles loading and storing prompt profiles. +// Repository is a transitional prompt-definition repository. +// It currently lives in internal/profile until package responsibilities +// are split in a follow-up refactor. type Repository interface { - GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) + GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) } diff --git a/internal/profile/repository_test.go b/internal/profile/repository_test.go index 3dee818..9753ce4 100644 --- a/internal/profile/repository_test.go +++ b/internal/profile/repository_test.go @@ -10,7 +10,7 @@ import ( "gitea.maximumdirect.net/eric/scriptorium/internal/domain" ) -func TestFilesystemRepository_GetProfile(t *testing.T) { +func TestFilesystemRepository_GetPromptDefinition(t *testing.T) { tmpDir, err := os.MkdirTemp("", "profile_test") if err != nil { t.Fatal(err) @@ -38,19 +38,19 @@ func TestFilesystemRepository_GetProfile(t *testing.T) { repo := NewFilesystemRepository(tmpDir) ctx := context.Background() - t.Run("valid profile", func(t *testing.T) { - p, err := repo.GetProfile(ctx, "test-profile", "") + t.Run("valid prompt definition", func(t *testing.T) { + p, err := repo.GetPromptDefinition(ctx, "test-profile", "") if err != nil { t.Fatalf("expected no error, got %v", err) } if p == nil || p.ID != "test-profile" { - t.Errorf("expected profile test-profile, got %v", p) + t.Errorf("expected prompt definition test-profile, got %v", p) } if p.Version != "1.0.0" { t.Fatalf("expected version 1.0.0, got %q", p.Version) } - if len(p.ExpectedInputs) != 2 || p.ExpectedInputs[0] != "transcript" || p.ExpectedInputs[1] != "glossary" { - t.Fatalf("unexpected expected_inputs: %#v", p.ExpectedInputs) + if len(p.Inputs) != 2 || p.Inputs[0].Name != "transcript" || p.Inputs[1].Name != "glossary" { + t.Fatalf("unexpected inputs: %#v", p.Inputs) } if len(p.Templates) != 2 { t.Fatalf("expected 2 templates, got %d", len(p.Templates)) @@ -64,48 +64,41 @@ func TestFilesystemRepository_GetProfile(t *testing.T) { if p.Validation.ValidationMode != domain.ValidationBasic { t.Fatalf("expected validation mode basic, got %q", p.Validation.ValidationMode) } - if p.ModelDefaults.TimeoutSeconds != 120 { - t.Fatalf("expected timeout_seconds 120, got %d", p.ModelDefaults.TimeoutSeconds) + if p.DefaultProfile != "test-exec" { + t.Fatalf("expected default profile test-exec, got %q", p.DefaultProfile) } }) t.Run("invalid YAML", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "invalid_yaml", "") + _, err := repo.GetPromptDefinition(ctx, "invalid_yaml", "") if !errors.Is(err, ErrInvalidYAML) { t.Errorf("expected ErrInvalidYAML, got %v", err) } }) t.Run("missing ID", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing-id", "") + _, err := repo.GetPromptDefinition(ctx, "missing-id", "") if !errors.Is(err, ErrProfileNotFound) { t.Errorf("expected ErrProfileNotFound for profile with missing ID, got %v", err) } }) t.Run("no templates", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "no-templates", "") + _, err := repo.GetPromptDefinition(ctx, "no-templates", "") if !errors.Is(err, ErrInvalidProfile) { t.Errorf("expected ErrInvalidProfile for profile with no templates, got %v", err) } }) t.Run("json schema mode missing schema path", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "json-schema-missing-path", "") + _, err := repo.GetPromptDefinition(ctx, "json-schema-missing-path", "") if !errors.Is(err, ErrInvalidProfile) { t.Errorf("expected ErrInvalidProfile for json_schema profile without schema_path, got %v", err) } }) - t.Run("negative timeout seconds", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "negative-timeout", "") - if !errors.Is(err, ErrInvalidProfile) { - t.Errorf("expected ErrInvalidProfile for negative timeout_seconds, got %v", err) - } - }) - - t.Run("profile not found", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "unknown", "") + t.Run("prompt definition not found", func(t *testing.T) { + _, err := repo.GetPromptDefinition(ctx, "unknown", "") if !errors.Is(err, ErrProfileNotFound) { t.Errorf("expected ErrProfileNotFound, got %v", err) } diff --git a/internal/profile/testdata/json_schema_missing_path.yaml b/internal/profile/testdata/json_schema_missing_path.yaml index be19267..2d1a589 100644 --- a/internal/profile/testdata/json_schema_missing_path.yaml +++ b/internal/profile/testdata/json_schema_missing_path.yaml @@ -1,5 +1,8 @@ id: json-schema-missing-path version: "1.0.0" +inputs: + - name: transcript + required: true templates: - role: user content: "Return JSON" diff --git a/internal/profile/testdata/missing_id.yaml b/internal/profile/testdata/missing_id.yaml index e91fc3a..21a928d 100644 --- a/internal/profile/testdata/missing_id.yaml +++ b/internal/profile/testdata/missing_id.yaml @@ -1,5 +1,8 @@ version: 1.0.0 description: Missing ID +inputs: + - name: transcript + required: true templates: - role: system content: Hello diff --git a/internal/profile/testdata/negative_timeout.yaml b/internal/profile/testdata/negative_timeout.yaml index 50fbc07..8f96b11 100644 --- a/internal/profile/testdata/negative_timeout.yaml +++ b/internal/profile/testdata/negative_timeout.yaml @@ -1,10 +1,11 @@ id: negative-timeout version: "1.0.0" +inputs: + - name: transcript + required: true templates: - role: user content: "Say hi" -model_defaults: - timeout_seconds: -1 output_format: text validation: validation_mode: none diff --git a/internal/profile/testdata/no_templates.yaml b/internal/profile/testdata/no_templates.yaml index 19fadd5..c36aa34 100644 --- a/internal/profile/testdata/no_templates.yaml +++ b/internal/profile/testdata/no_templates.yaml @@ -1,5 +1,8 @@ id: no-templates version: 1.0.0 +inputs: + - name: transcript + required: true templates: [] output_format: text validation: diff --git a/internal/profile/testdata/valid.yaml b/internal/profile/testdata/valid.yaml index 566e67f..b8d2318 100644 --- a/internal/profile/testdata/valid.yaml +++ b/internal/profile/testdata/valid.yaml @@ -1,18 +1,19 @@ id: test-profile version: "1.0.0" -description: A valid test profile -expected_inputs: - - transcript - - glossary +default_profile: test-exec +description: A valid test prompt definition +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml templates: - role: system content: "You are a helpful assistant." - role: user content: 'Analyze this: {{input "transcript"}}' -model_defaults: - model: gpt-4o - temperature: 0.7 - timeout_seconds: 120 output_format: markdown validation: validation_mode: basic diff --git a/internal/prompt/go_renderer.go b/internal/prompt/go_renderer.go index e96f3e0..241ff50 100644 --- a/internal/prompt/go_renderer.go +++ b/internal/prompt/go_renderer.go @@ -23,16 +23,19 @@ func NewGoRenderer() Renderer { return &goRenderer{} } -func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { - if profile == nil { - return nil, fmt.Errorf("%w: nil profile", ErrRenderFailure) +func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { + if definition == nil { + return nil, fmt.Errorf("%w: nil prompt definition", ErrRenderFailure) } // 1. Verify required inputs - for _, req := range profile.ExpectedInputs { - art, ok := inputs[req] + for _, in := range definition.Inputs { + if !in.Required { + continue + } + art, ok := inputs[in.Name] if !ok || art == nil { - return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, req) + return nil, fmt.Errorf("%w: %s", ErrMissingRequiredInput, in.Name) } } @@ -49,7 +52,7 @@ func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile, var renderedMessages []domain.RenderedMessage - for i, tmplMsg := range profile.Templates { + for i, tmplMsg := range definition.Templates { select { case <-ctx.Done(): return nil, ctx.Err() @@ -61,6 +64,9 @@ func (r *goRenderer) Render(ctx context.Context, profile *domain.PromptProfile, } // Parse and execute template + if tmplMsg.ContentFile != "" { + return nil, fmt.Errorf("%w: message %d: content_file is not implemented yet", ErrRenderFailure, i) + } tmpl, err := template.New(fmt.Sprintf("msg_%d", i)).Funcs(funcs).Option("missingkey=error").Parse(tmplMsg.Content) if err != nil { return nil, fmt.Errorf("%w: message %d: %v", ErrInvalidTemplate, i, err) diff --git a/internal/prompt/renderer.go b/internal/prompt/renderer.go index 815ebeb..9e4469d 100644 --- a/internal/prompt/renderer.go +++ b/internal/prompt/renderer.go @@ -7,5 +7,5 @@ import ( // Renderer renders prompt templates using named artifacts and variables. type Renderer interface { - Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) + Render(ctx context.Context, definition *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) } diff --git a/internal/prompt/renderer_test.go b/internal/prompt/renderer_test.go index e129db0..5925366 100644 --- a/internal/prompt/renderer_test.go +++ b/internal/prompt/renderer_test.go @@ -12,9 +12,9 @@ func TestGoRenderer_Render(t *testing.T) { renderer := NewGoRenderer() ctx := context.Background() - profile := &domain.PromptProfile{ + profile := &domain.PromptDefinition{ ID: "test-profile", - ExpectedInputs: []string{"transcript"}, + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{ {Role: "system", Content: "You are a {{.role}}."}, {Role: "user", Content: "Analyze this: {{input \"transcript\"}}"}, @@ -54,7 +54,8 @@ func TestGoRenderer_Render(t *testing.T) { }) t.Run("unknown input in template", func(t *testing.T) { - profileUnknown := &domain.PromptProfile{ + profileUnknown := &domain.PromptDefinition{ + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{ {Role: "user", Content: "Hello {{input \"ghost\"}}"}, }, @@ -69,7 +70,8 @@ func TestGoRenderer_Render(t *testing.T) { }) t.Run("invalid template syntax", func(t *testing.T) { - profileInvalid := &domain.PromptProfile{ + profileInvalid := &domain.PromptDefinition{ + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{ {Role: "user", Content: "Hello {{.unclosed"}, }, @@ -81,7 +83,8 @@ func TestGoRenderer_Render(t *testing.T) { }) t.Run("empty message role", func(t *testing.T) { - profileNoRole := &domain.PromptProfile{ + profileNoRole := &domain.PromptDefinition{ + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{ {Role: "", Content: "Hello"}, }, @@ -93,7 +96,8 @@ func TestGoRenderer_Render(t *testing.T) { }) t.Run("missing variable in template", func(t *testing.T) { - profileMissingVar := &domain.PromptProfile{ + profileMissingVar := &domain.PromptDefinition{ + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{ {Role: "system", Content: "You are {{.missing}}"}, }, diff --git a/internal/usecase/integration_test.go b/internal/usecase/integration_test.go index 455e975..f3f943e 100644 --- a/internal/usecase/integration_test.go +++ b/internal/usecase/integration_test.go @@ -44,7 +44,12 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) { ) res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "generic.structured_events", + PromptID: "generic.structured_events", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{ + Endpoint: "http://llm/v1", + Model: "test-model", + }, Inputs: map[string]domain.ArtifactRef{ "transcript": { Type: domain.ArtifactRefFile, @@ -60,17 +65,17 @@ func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) { t.Fatalf("expected no error, got %v", err) } - if res.ProfileID != "generic.structured_events" { - t.Fatalf("unexpected profile id: %q", res.ProfileID) + if res.PromptID != "generic.structured_events" { + t.Fatalf("unexpected prompt id: %q", res.PromptID) } if res.RunID == "" { t.Fatal("expected run id") } - if res.ProfileHash == "" { - t.Fatal("expected profile hash") + if res.PromptHash == "" { + t.Fatal("expected prompt hash") } - if res.ProfileVersion != "1.0.0" { - t.Fatalf("unexpected profile version: %q", res.ProfileVersion) + if res.PromptVersion != "1.0.0" { + t.Fatalf("unexpected prompt version: %q", res.PromptVersion) } if res.Validation.Status != domain.ValidationPassed { t.Fatalf("expected passed validation, got %q", res.Validation.Status) diff --git a/internal/usecase/repairer.go b/internal/usecase/repairer.go index ac23098..84c2af5 100644 --- a/internal/usecase/repairer.go +++ b/internal/usecase/repairer.go @@ -17,7 +17,7 @@ type OutputRepairer interface { type RepairRequest struct { PreviousOutput string ValidationErrors []string - Target domain.ModelTarget + Target domain.ExecutionTarget Attempt int MaxAttempts int Mode domain.ValidationMode diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 2a7cc6b..798acb3 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -21,7 +21,7 @@ import ( var ( ErrInvalidRequest = errors.New("invalid run request") - ErrProfileLoad = errors.New("failed to load profile") + ErrProfileLoad = errors.New("failed to load prompt definition") ErrArtifactLoad = errors.New("failed to load artifact") ErrPromptRender = errors.New("failed to render prompt") ErrLLMGenerate = errors.New("failed to generate output") @@ -67,8 +67,8 @@ func NewRunnerWithRepairer( } func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunResult, error) { - if strings.TrimSpace(req.ProfileID) == "" { - return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest) + if strings.TrimSpace(req.PromptID) == "" { + return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest) } runID, err := newRunID() @@ -78,17 +78,32 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes start := time.Now().UTC() - prof, err := r.profiles.GetProfile(ctx, req.ProfileID, req.ProfileVersion) + def, err := r.profiles.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) } - profileHash, err := hashProfile(prof) + promptDefinitionHash, err := hashPromptDefinition(def) if err != nil { - return nil, fmt.Errorf("%w: failed to hash profile: %v", ErrProfileLoad, err) + return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrProfileLoad, err) } - - effectiveModel := mergeModelTarget(prof.ModelDefaults, req.Model) - effectiveContract := resolveOutputContract(prof, req.Validation) + selectedProfileID := strings.TrimSpace(req.ProfileID) + if selectedProfileID == "" { + selectedProfileID = strings.TrimSpace(def.DefaultProfile) + } + if selectedProfileID == "" { + return nil, fmt.Errorf("%w: profile id is required either in request or prompt default_profile", ErrInvalidRequest) + } + if req.Execution == nil { + return nil, fmt.Errorf("%w: execution override is required until execution profile loading is implemented", ErrInvalidRequest) + } + effectiveModel := mergeExecutionTarget(domain.ExecutionTarget{}, req.Execution) + if strings.TrimSpace(effectiveModel.Endpoint) == "" { + return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest) + } + if strings.TrimSpace(effectiveModel.Model) == "" { + return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest) + } + effectiveContract := resolveOutputContract(def, req.Validation) resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs)) inputHashes := make(map[string]string, len(req.Inputs)) @@ -104,12 +119,12 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes inputHashes[name] = art.Hash } - renderedPrompt, err := r.renderer.Render(ctx, prof, resolvedInputs, req.Vars) + renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars) if err != nil { return nil, fmt.Errorf("%w: %w", ErrPromptRender, err) } - promptHash := hashRenderedPrompt(*renderedPrompt) + renderedPromptHash := hashRenderedPrompt(*renderedPrompt) genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{ Prompt: *renderedPrompt, @@ -158,22 +173,23 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes end := time.Now().UTC() return &domain.RunResult{ - RunID: runID, - Artifact: outputArtifact, - RawOutput: genResp.Content, - Validation: validationResult, - ProfileID: prof.ID, - ProfileVersion: prof.Version, - ProfileHash: profileHash, - ModelName: effectiveModel.Model, - Endpoint: effectiveModel.Endpoint, - ModelParams: effectiveModel, - InputHashes: inputHashes, - PromptHash: promptHash, - Usage: genResp.Usage, - StartTime: start, - EndTime: end, - Duration: end.Sub(start), + RunID: runID, + Artifact: outputArtifact, + RawOutput: genResp.Content, + Validation: validationResult, + PromptID: def.ID, + PromptVersion: def.Version, + PromptHash: promptDefinitionHash, + RenderedPromptHash: renderedPromptHash, + SelectedProfileID: selectedProfileID, + ModelName: effectiveModel.Model, + Endpoint: effectiveModel.Endpoint, + EffectiveModelParams: effectiveModel, + InputHashes: inputHashes, + Usage: genResp.Usage, + StartTime: start, + EndTime: end, + Duration: end.Sub(start), }, nil } @@ -209,7 +225,7 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema } -func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) domain.ModelTarget { +func mergeExecutionTarget(base domain.ExecutionTarget, override *domain.ExecutionTarget) domain.ExecutionTarget { if override == nil { return base } @@ -233,13 +249,26 @@ func mergeModelTarget(base domain.ModelTarget, override *domain.ModelTarget) dom if override.TimeoutSeconds != 0 { out.TimeoutSeconds = override.TimeoutSeconds } + if strings.TrimSpace(override.ReasoningEffort) != "" { + out.ReasoningEffort = override.ReasoningEffort + } + if strings.TrimSpace(override.APIKeyEnv) != "" { + out.APIKeyEnv = override.APIKeyEnv + } + if len(override.ExtraParams) > 0 { + cp := make(map[string]string, len(override.ExtraParams)) + for k, v := range override.ExtraParams { + cp[k] = v + } + out.ExtraParams = cp + } return out } -func resolveOutputContract(prof *domain.PromptProfile, override *domain.OutputContract) domain.OutputContract { - contract := prof.Validation +func resolveOutputContract(def *domain.PromptDefinition, override *domain.OutputContract) domain.OutputContract { + contract := def.Validation if contract.Format == "" { - contract.Format = prof.OutputFormat + contract.Format = def.OutputFormat } if override != nil { contract = *override @@ -283,8 +312,8 @@ func buildOutputArtifact(content string, format domain.OutputFormat) domain.Arti } } -func hashProfile(prof *domain.PromptProfile) (string, error) { - b, err := json.Marshal(prof) +func hashPromptDefinition(def *domain.PromptDefinition) (string, error) { + b, err := json.Marshal(def) if err != nil { return "", err } diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 199f39d..b3c4638 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -5,8 +5,6 @@ import ( "crypto/sha256" "encoding/hex" "errors" - "os" - "path/filepath" "regexp" "testing" @@ -14,20 +12,20 @@ import ( "gitea.maximumdirect.net/eric/scriptorium/internal/validate" ) -type fakeProfileRepo struct { - profile *domain.PromptProfile +type fakePromptRepo struct { + def *domain.PromptDefinition err error lastID string lastVersion string } -func (f *fakeProfileRepo) GetProfile(ctx context.Context, id string, version string) (*domain.PromptProfile, error) { +func (f *fakePromptRepo) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { f.lastID = id f.lastVersion = version if f.err != nil { return nil, f.err } - return f.profile, nil + return f.def, nil } type fakeArtifactReader struct { @@ -40,8 +38,8 @@ func (f *fakeArtifactReader) Read(ctx context.Context, ref domain.ArtifactRef) ( return nil, err } if art, ok := f.artifactsByURI[ref.URI]; ok { - copy := *art - return ©, nil + cp := *art + return &cp, nil } return nil, errors.New("artifact not found") } @@ -51,7 +49,7 @@ type fakeRenderer struct { err error } -func (f *fakeRenderer) Render(ctx context.Context, profile *domain.PromptProfile, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { +func (f *fakeRenderer) Render(ctx context.Context, def *domain.PromptDefinition, inputs map[string]*domain.Artifact, vars map[string]string) (*domain.RenderedPrompt, error) { if f.err != nil { return nil, f.err } @@ -73,15 +71,11 @@ func (f *fakeLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*do } type fakeValidator struct { - result domain.ValidationResult - err error - called bool - lastContract domain.OutputContract + result domain.ValidationResult + err error } func (f *fakeValidator) Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error) { - f.called = true - f.lastContract = contract if f.err != nil { return domain.ValidationResult{}, f.err } @@ -92,12 +86,10 @@ type fakeRepairer struct { responses []*domain.GenerateResponse err error calls int - lastReq RepairRequest } func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error) { f.calls++ - f.lastReq = req if f.err != nil { return nil, f.err } @@ -112,156 +104,83 @@ func (f *fakeRepairer) Repair(ctx context.Context, req RepairRequest) (*domain.G } func TestRunnerRunSuccessful(t *testing.T) { - repo := &fakeProfileRepo{ - profile: &domain.PromptProfile{ - ID: "p1", - Version: "1.0.0", - OutputFormat: domain.FormatMarkdown, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep1", - Model: "model-default", - Temperature: 0.4, - MaxTokens: 200, - TopP: 0.9, - TimeoutSeconds: 90, - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationBasic, - }, - }, - } - + repo := &fakePromptRepo{def: promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)} reader := &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ - "a://t": {Body: []byte("transcript body"), Hash: hashString("transcript body")}, - "a://g": {Body: []byte("glossary body"), Hash: hashString("glossary body")}, - }} - - renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{ - {Role: "system", Content: "System context"}, - {Role: "user", Content: "Please summarize"}, - }}} - - llmClient := &fakeLLM{resp: &domain.GenerateResponse{ - Content: "# recap\n- item", - Usage: domain.TokenUsage{ - PromptTokens: 10, - CompletionTokens: 20, - TotalTokens: 30, - }, + "a://t": {Body: []byte("transcript"), Hash: hashString("transcript")}, + "a://g": {Body: []byte("glossary"), Hash: hashString("glossary")}, }} + renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} + llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} runner := NewRunner(repo, reader, renderer, llmClient, nil) - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p1", - ProfileVersion: "1.0.0", + PromptID: "p", + PromptVersion: "1", + ProfileID: "exec", Inputs: map[string]domain.ArtifactRef{ "transcript": {Type: domain.ArtifactRefFile, URI: "a://t"}, "glossary": {Type: domain.ArtifactRefFile, URI: "a://g"}, }, - Model: &domain.ModelTarget{ - Model: "model-override", - Temperature: 0, - MaxTokens: 0, - TimeoutSeconds: 0, - }, + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", Temperature: 0.3, TimeoutSeconds: 90}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } - - if res.ProfileID != "p1" || res.ProfileVersion != "1.0.0" { - t.Fatalf("unexpected profile metadata: id=%q version=%q", res.ProfileID, res.ProfileVersion) + if res.PromptID != "p" || res.PromptVersion != "1" { + t.Fatalf("unexpected prompt metadata: %+v", res) + } + if res.SelectedProfileID != "exec" { + t.Fatalf("expected selected profile exec, got %q", res.SelectedProfileID) } if ok, _ := regexp.MatchString(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`, res.RunID); !ok { - t.Fatalf("expected UUIDv4 run id, got %q", res.RunID) + t.Fatalf("invalid run id: %q", res.RunID) } - if res.ProfileHash == "" { - t.Fatal("expected non-empty profile hash") + if res.PromptHash == "" || res.RenderedPromptHash == "" { + t.Fatal("expected prompt hashes") } - if res.ModelName != "model-override" { - t.Fatalf("expected model override to apply, got %q", res.ModelName) + if res.EffectiveModelParams.Model != "m" || res.Endpoint != "http://llm/v1" { + t.Fatalf("unexpected model params: %+v", res.EffectiveModelParams) } - if res.Endpoint != "ep1" { - t.Fatalf("expected endpoint from profile default, got %q", res.Endpoint) - } - if res.Artifact.ContentType != "text/markdown" { - t.Fatalf("expected markdown content type, got %q", res.Artifact.ContentType) - } - if string(res.Artifact.Body) != "# recap\n- item" { - t.Fatalf("unexpected artifact body: %q", string(res.Artifact.Body)) - } - if res.RawOutput != "# recap\n- item" { - t.Fatalf("expected raw output preserved, got %q", res.RawOutput) + if res.RawOutput != "# recap" { + t.Fatalf("expected raw output, got %q", res.RawOutput) } if res.Validation.Status != domain.ValidationSkipped { t.Fatalf("expected skipped validation, got %q", res.Validation.Status) } - if res.Validation.Mode != domain.ValidationBasic { - t.Fatalf("expected validation mode basic in skipped result, got %q", res.Validation.Mode) - } - if res.PromptHash == "" { - t.Fatal("expected non-empty prompt hash") - } - if res.Usage.TotalTokens != 30 { - t.Fatalf("expected usage to propagate, got %+v", res.Usage) - } - if res.StartTime.IsZero() || res.EndTime.IsZero() { - t.Fatal("expected start and end times") - } - if res.EndTime.Before(res.StartTime) { - t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) - } - if res.Duration < 0 { - t.Fatalf("expected non-negative duration, got %s", res.Duration) - } - - if got := res.InputHashes["transcript"]; got != hashString("transcript body") { - t.Fatalf("unexpected transcript hash: %q", got) - } - if got := res.InputHashes["glossary"]; got != hashString("glossary body") { - t.Fatalf("unexpected glossary hash: %q", got) - } - - if llmClient.lastReq.Target.Temperature != 0.4 { - t.Fatalf("expected zero-valued request field not to override default temperature, got %v", llmClient.lastReq.Target.Temperature) - } if llmClient.lastReq.Target.TimeoutSeconds != 90 { - t.Fatalf("expected zero-valued request timeout not to override default timeout, got %d", llmClient.lastReq.Target.TimeoutSeconds) - } - if res.ModelParams.Model != "model-override" || res.ModelParams.Endpoint != "ep1" { - t.Fatalf("expected effective model params in result, got %+v", res.ModelParams) + t.Fatalf("expected timeout propagation, got %d", llmClient.lastReq.Target.TimeoutSeconds) } } -func TestRunnerRunProfileLoadFailure(t *testing.T) { - runner := NewRunner( - &fakeProfileRepo{err: errors.New("boom")}, - &fakeArtifactReader{}, - &fakeRenderer{}, - &fakeLLM{}, - nil, - ) - - _, err := runner.Run(context.Background(), domain.RunRequest{ProfileID: "p"}) +func TestRunnerRunPromptLoadFailure(t *testing.T) { + runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) + _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrProfileLoad) { t.Fatalf("expected ErrProfileLoad, got %v", err) } } -func TestMergeModelTargetTimeoutOverride(t *testing.T) { - base := domain.ModelTarget{TimeoutSeconds: 30} - override := &domain.ModelTarget{TimeoutSeconds: 75} +func TestRunnerRunMissingProfileSelection(t *testing.T) { + repo := &fakePromptRepo{def: &domain.PromptDefinition{ID: "p", Version: "1", Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, OutputFormat: domain.FormatText, Validation: domain.OutputContract{ValidationMode: domain.ValidationNone}}} + runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) + _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected invalid request, got %v", err) + } +} - got := mergeModelTarget(base, override) - if got.TimeoutSeconds != 75 { - t.Fatalf("expected timeout override to apply, got %d", got.TimeoutSeconds) +func TestRunnerRunMissingExecutionOverride(t *testing.T) { + repo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + runner := NewRunner(repo, &fakeArtifactReader{}, &fakeRenderer{}, &fakeLLM{}, nil) + _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec"}) + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected invalid request, got %v", err) } } func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( - &fakeProfileRepo{profile: minimalProfile()}, + &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, @@ -269,10 +188,10 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) { ) _, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}, - }, + PromptID: "p", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"}, + Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://bad"}}, }) if !errors.Is(err, ErrArtifactLoad) { t.Fatalf("expected ErrArtifactLoad, got %v", err) @@ -281,18 +200,17 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) { func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( - &fakeProfileRepo{profile: minimalProfile()}, + &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{err: errors.New("render failed")}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, ) - _, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, + PromptID: "p", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"}, + Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if !errors.Is(err, ErrPromptRender) { t.Fatalf("expected ErrPromptRender, got %v", err) @@ -301,405 +219,82 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) { func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( - &fakeProfileRepo{profile: minimalProfile()}, + &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{err: errors.New("llm failed")}, nil, ) - _, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, + PromptID: "p", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"}, + Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if !errors.Is(err, ErrLLMGenerate) { t.Fatalf("expected ErrLLMGenerate, got %v", err) } } -func TestRunnerRunValidationFailureNonError(t *testing.T) { - validator := &fakeValidator{result: domain.ValidationResult{ - Status: domain.ValidationFailed, - Mode: domain.ValidationBasic, - Errors: []string{"bad output"}, - IsValid: false, - }} - +func TestRunnerRunValidationFailurePreservesRawOutput(t *testing.T) { + validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} runner := NewRunner( - &fakeProfileRepo{profile: minimalProfile()}, + &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, validator, ) - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, + PromptID: "p", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"}, + Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } - if res.Validation.Status != domain.ValidationFailed { - t.Fatalf("expected validation failed result, got %q", res.Validation.Status) - } - if res.RawOutput != "raw output" { - t.Fatalf("expected raw output preserved, got %q", res.RawOutput) - } - if !validator.called { - t.Fatal("expected validator to be called") + if res.Validation.Status != domain.ValidationFailed || res.RawOutput != "raw output" { + t.Fatalf("unexpected validation/raw output: %+v", res) } } -func TestRunnerRunValidationRuntimeError(t *testing.T) { - validator := &fakeValidator{err: errors.New("validator unavailable")} - - runner := NewRunner( - &fakeProfileRepo{profile: minimalProfile()}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, - validator, - ) - - _, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if !errors.Is(err, ErrValidation) { - t.Fatalf("expected ErrValidation, got %v", err) - } -} - -func TestRunnerRunValidationFailureWithRealValidatorPreservesRawOutput(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - runner := NewRunner( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "schema.json", - Format: domain.FormatJSON, - }, - }}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}}, - validate.NewStandardValidator(tmp), - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Validation.Status != domain.ValidationFailed { - t.Fatalf("expected validation failed, got %q", res.Validation.Status) - } - if res.RawOutput != `{"count":1}` { - t.Fatalf("expected raw output preserved, got %q", res.RawOutput) - } -} - -func TestRunnerRunNoRepairWhenDisabled(t *testing.T) { - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{{Content: `{"ok":true}`}}, - } - +func TestRunnerRunRepairBounded(t *testing.T) { + repairer := &fakeRepairer{responses: []*domain.GenerateResponse{{Content: `{"broken":`}, {Content: `{"still":`}}} runner := NewRunnerWithRepairer( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - Format: domain.FormatJSON, - RepairAttempts: 0, - }, - }}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}}, - validate.NewStandardValidator(t.TempDir()), - repairer, - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if repairer.calls != 0 { - t.Fatalf("expected no repair calls, got %d", repairer.calls) - } - if res.Validation.Status != domain.ValidationFailed { - t.Fatalf("expected failed validation, got %q", res.Validation.Status) - } - if res.RawOutput != `{"broken":` { - t.Fatalf("expected original output preserved, got %q", res.RawOutput) - } - if res.Validation.RepairAttempts != 0 { - t.Fatalf("expected repair attempts 0, got %d", res.Validation.RepairAttempts) - } -} - -func TestRunnerRunSuccessfulRepairAfterInvalidJSON(t *testing.T) { - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{{Content: `{"ok":true}`, Usage: domain.TokenUsage{TotalTokens: 5}}}, - } - - runner := NewRunnerWithRepairer( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - Format: domain.FormatJSON, - RepairAttempts: 1, - }, - }}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`, Usage: domain.TokenUsage{TotalTokens: 3}}}, - validate.NewStandardValidator(t.TempDir()), - repairer, - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if repairer.calls != 1 { - t.Fatalf("expected one repair call, got %d", repairer.calls) - } - if res.Validation.Status != domain.ValidationPassed { - t.Fatalf("expected passed validation, got %q", res.Validation.Status) - } - if res.Validation.RepairAttempts != 1 { - t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) - } - if res.RawOutput != `{"ok":true}` { - t.Fatalf("expected repaired output, got %q", res.RawOutput) - } -} - -func TestRunnerRunSuccessfulRepairAfterSchemaFailure(t *testing.T) { - tmp := t.TempDir() - if err := os.WriteFile(filepath.Join(tmp, "schema.json"), []byte(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["name"], - "properties": { - "name": {"type": "string"} - } -}`), 0644); err != nil { - t.Fatal(err) - } - - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{{Content: `{"name":"eris"}`}}, - } - - runner := NewRunnerWithRepairer( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSONSchema, - SchemaPath: "schema.json", - Format: domain.FormatJSON, - RepairAttempts: 1, - }, - }}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"count":1}`}}, - validate.NewStandardValidator(tmp), - repairer, - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if res.Validation.Status != domain.ValidationPassed { - t.Fatalf("expected passed validation, got %q", res.Validation.Status) - } - if res.Validation.RepairAttempts != 1 { - t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) - } - if res.RawOutput != `{"name":"eris"}` { - t.Fatalf("expected repaired output, got %q", res.RawOutput) - } -} - -func TestRunnerRunFailedRepairPreservesRawOutputAndErrors(t *testing.T) { - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{ - {Content: `{"repair1":`}, - {Content: `{"repair2":`}, - }, - } - - runner := NewRunnerWithRepairer( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - Format: domain.FormatJSON, - RepairAttempts: 2, - }, - }}, + &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}}, - validate.NewStandardValidator(t.TempDir()), + validate.NewStandardValidator("."), repairer, ) - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, + PromptID: "p", + ProfileID: "exec", + Execution: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m"}, + Inputs: map[string]domain.ArtifactRef{"transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}}, }) if err != nil { t.Fatalf("expected no error, got %v", err) } - if res.Validation.Status != domain.ValidationFailed { - t.Fatalf("expected failed validation, got %q", res.Validation.Status) - } - if len(res.Validation.Errors) == 0 { - t.Fatal("expected validation errors after failed repair") - } - if res.Validation.RepairAttempts != 2 { - t.Fatalf("expected repair attempts 2, got %d", res.Validation.RepairAttempts) - } - if res.RawOutput != `{"repair2":` { - t.Fatalf("expected final repaired output preserved, got %q", res.RawOutput) + if repairer.calls != 1 || res.Validation.RepairAttempts != 1 { + t.Fatalf("expected one bounded repair, calls=%d attempts=%d", repairer.calls, res.Validation.RepairAttempts) } } -func TestRunnerRunRepairAttemptsBounded(t *testing.T) { - repairer := &fakeRepairer{ - responses: []*domain.GenerateResponse{ - {Content: `{"repair1":`}, - {Content: `{"repair2":`}, - {Content: `{"repair3":`}, - }, - } - - runner := NewRunnerWithRepairer( - &fakeProfileRepo{profile: &domain.PromptProfile{ - ID: "p-json", - Version: "1", - OutputFormat: domain.FormatJSON, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, - Validation: domain.OutputContract{ - ValidationMode: domain.ValidationJSON, - Format: domain.FormatJSON, - RepairAttempts: 1, - }, - }}, - &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{"a://ok": {Body: []byte("x"), Hash: hashString("x")}}}, - &fakeRenderer{rendered: &domain.RenderedPrompt{}}, - &fakeLLM{resp: &domain.GenerateResponse{Content: `{"initial":`}}, - validate.NewStandardValidator(t.TempDir()), - repairer, - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - ProfileID: "p-json", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://ok"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if repairer.calls != 1 { - t.Fatalf("expected repair calls bounded to 1, got %d", repairer.calls) - } - if res.Validation.RepairAttempts != 1 { - t.Fatalf("expected repair attempts 1, got %d", res.Validation.RepairAttempts) - } -} - -func minimalProfile() *domain.PromptProfile { - return &domain.PromptProfile{ - ID: "p", - Version: "1", - OutputFormat: domain.FormatText, - ModelDefaults: domain.ModelTarget{ - Endpoint: "ep", - Model: "m", - }, +func promptDef(format domain.OutputFormat, mode domain.ValidationMode, attempts int) *domain.PromptDefinition { + return &domain.PromptDefinition{ + ID: "p", + Version: "1", + DefaultProfile: "exec", + Inputs: []domain.PromptInput{{Name: "transcript", Required: true}}, + Templates: []domain.PromptMessageTemplate{{Role: "user", Content: "x"}}, + OutputFormat: format, Validation: domain.OutputContract{ - ValidationMode: domain.ValidationBasic, + ValidationMode: mode, + RepairAttempts: attempts, + Format: format, }, } } diff --git a/profiles/dnd.session_recap.yaml b/profiles/dnd.session_recap.yaml index 6211f35..5e7907e 100644 --- a/profiles/dnd.session_recap.yaml +++ b/profiles/dnd.session_recap.yaml @@ -1,9 +1,14 @@ id: dnd.session_recap version: "1.0.0" -description: Example D&D session recap profile for demonstration only. -expected_inputs: - - transcript - - glossary +default_profile: local-default +description: Example D&D session recap prompt definition for demonstration only. +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml templates: - role: system content: | @@ -21,11 +26,6 @@ templates: Glossary: {{input "glossary"}} -model_defaults: - endpoint: http://localhost:8000/v1 - model: gpt-4o-mini - temperature: 0.3 - max_tokens: 900 output_format: markdown validation: validation_mode: basic diff --git a/profiles/generic.markdown_summary.yaml b/profiles/generic.markdown_summary.yaml index b094081..caf7c5d 100644 --- a/profiles/generic.markdown_summary.yaml +++ b/profiles/generic.markdown_summary.yaml @@ -1,9 +1,16 @@ id: generic.markdown_summary version: "1.0.0" +default_profile: local-default description: Generic markdown summary from transcript and glossary. -expected_inputs: - - transcript - - glossary +inputs: + - name: transcript + required: true + content_type: text/markdown + description: Source transcript content + - name: glossary + required: false + content_type: text/yaml + description: Optional glossary context templates: - role: system content: | @@ -18,11 +25,6 @@ templates: Reference glossary: {{input "glossary"}} -model_defaults: - endpoint: http://localhost:8000/v1 - model: gpt-4o-mini - temperature: 0.2 - max_tokens: 700 output_format: markdown validation: validation_mode: basic diff --git a/profiles/generic.structured_events.yaml b/profiles/generic.structured_events.yaml index 2722a49..d873280 100644 --- a/profiles/generic.structured_events.yaml +++ b/profiles/generic.structured_events.yaml @@ -1,9 +1,14 @@ id: generic.structured_events version: "1.0.0" +default_profile: local-default description: Produce structured event JSON from a transcript. -expected_inputs: - - transcript - - glossary +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml templates: - role: system content: | @@ -18,11 +23,6 @@ templates: Glossary: {{input "glossary"}} -model_defaults: - endpoint: http://localhost:8000/v1 - model: gpt-4o-mini - temperature: 0.0 - max_tokens: 500 output_format: json validation: format: json