From 328703df36f30c57fba66574ef04c5555c3c3a4b Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 4 May 2026 21:23:48 -0500 Subject: [PATCH] Add example profiles, schemas, and an end-to-end local fixture --- README.md | 101 +++++++++++++++++++++--- examples/fixtures/glossary.yml | 7 ++ examples/fixtures/transcript.md | 7 ++ internal/usecase/integration_test.go | 99 +++++++++++++++++++++++ profiles/dnd.session_recap.yaml | 31 ++++++++ profiles/generic.markdown_summary.yaml | 28 +++++++ profiles/generic.structured_events.yaml | 31 ++++++++ schemas/structured_events.schema.json | 27 +++++++ 8 files changed, 319 insertions(+), 12 deletions(-) create mode 100644 examples/fixtures/glossary.yml create mode 100644 examples/fixtures/transcript.md create mode 100644 internal/usecase/integration_test.go create mode 100644 profiles/dnd.session_recap.yaml create mode 100644 profiles/generic.markdown_summary.yaml create mode 100644 profiles/generic.structured_events.yaml create mode 100644 schemas/structured_events.schema.json diff --git a/README.md b/README.md index 570f775..d7d178c 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,102 @@ # scriptorium -Scriptorium is a prompt-profile execution engine written in Go. +Scriptorium (Analyzer) is a generic prompt-profile execution engine written in Go. -Current implementation scope: -- domain model and core interfaces -- YAML-backed prompt profile loading -- inline and file artifact reading -- provider-neutral prompt rendering +It loads a prompt profile, resolves named input artifacts, renders a prompt, calls an OpenAI-compatible LLM endpoint, validates output, and returns an artifact plus metadata. -## Local CLI usage +## Relationship to Narratio -Run a profile locally against an OpenAI-compatible endpoint: +In the broader workflow, Narratio handles pipeline orchestration (transcription, cleanup, storage, notifications). Scriptorium handles only prompt-profile execution for a single run. + +## Repository Example Assets + +- Profiles: `profiles/` +- Schemas: `schemas/` +- Tiny fixtures: `examples/fixtures/` + +Included profiles: +- `generic.markdown_summary` +- `dnd.session_recap` (example content only; no D&D-specific Go logic) +- `generic.structured_events` (JSON + JSON Schema validation) + +## Run a Local Profile (CLI) ```bash go run ./cmd/scriptorium run \ --profile-dir ./profiles \ - --profile-id recap \ - --input transcript=./testdata/transcript.md \ - --input glossary=./testdata/glossary.yml \ - --var session_date=2026-05-04 \ + --profile-id generic.markdown_summary \ + --input transcript=./examples/fixtures/transcript.md \ + --input glossary=./examples/fixtures/glossary.yml \ --llm-base-url http://localhost:8000/v1 \ --model gpt-4o-mini \ --out ./out.md ``` + +For schema-validated JSON output: + +```bash +go run ./cmd/scriptorium run \ + --profile-dir ./profiles \ + --profile-id generic.structured_events \ + --input transcript=./examples/fixtures/transcript.md \ + --input glossary=./examples/fixtures/glossary.yml \ + --llm-base-url http://localhost:8000/v1 \ + --model gpt-4o-mini \ + --schema-dir ./schemas \ + --out ./events.json +``` + +## Start Local HTTP API + +```bash +go run ./cmd/scriptorium serve \ + --addr :8080 \ + --profile-dir ./profiles \ + --schema-dir ./schemas \ + --llm-base-url http://localhost:8000/v1 \ + --model gpt-4o-mini +``` + +## Call `POST /v1/runs` + +```bash +curl -sS http://localhost:8080/v1/runs \ + -H 'Content-Type: application/json' \ + -d '{ + "profile_id": "generic.structured_events", + "inputs": { + "transcript": {"type": "file", "uri": "./examples/fixtures/transcript.md"}, + "glossary": {"type": "file", "uri": "./examples/fixtures/glossary.yml"} + }, + "model": {"model": "gpt-4o-mini"} + }' +``` + +Response shape: +- `artifact` +- `validation` +- `metadata` +- `raw_model_output` + +## Add a New Prompt Profile + +1. Add a YAML file under `profiles/` with: + - `id`, `version`, `expected_inputs`, `templates`, `model_defaults`, `output_format`, `validation` +2. Use template helpers such as `{{input "transcript"}}`. +3. For structured JSON output, set: + - `output_format: json` + - `validation.validation_mode: json_schema` + - `validation.schema_path: ` +4. Place schema files in `schemas/` and pass `--schema-dir ./schemas` for CLI/serve. + +## Validation Behavior + +Validation modes currently implemented: +- `none` +- `basic` (non-empty output) +- `json` (must parse as JSON) +- `json_schema` (must parse JSON and satisfy schema) + +Important behavior: +- Validation content failures are returned as structured run results (`validation.status = failed`) and preserve `raw_model_output`. +- Validation runtime/configuration failures are treated as run errors. diff --git a/examples/fixtures/glossary.yml b/examples/fixtures/glossary.yml new file mode 100644 index 0000000..dd5c8cd --- /dev/null +++ b/examples/fixtures/glossary.yml @@ -0,0 +1,7 @@ +party: + - Rin (ranger) + - Kara (cleric) +locations: + - Ruined Watchtower +factions: + - Goblin Raiders diff --git a/examples/fixtures/transcript.md b/examples/fixtures/transcript.md new file mode 100644 index 0000000..1b3e3c9 --- /dev/null +++ b/examples/fixtures/transcript.md @@ -0,0 +1,7 @@ +# Session Transcript (Excerpt) + +DM: The party arrives at the ruined watchtower at dusk. +Rin: I search the courtyard for tracks. +DM: You find fresh boot prints leading below the tower. +Kara: We light a lantern and descend carefully. +DM: In the cellar, a frightened scout begs for help escaping goblin raiders. diff --git a/internal/usecase/integration_test.go b/internal/usecase/integration_test.go new file mode 100644 index 0000000..97cfd0d --- /dev/null +++ b/internal/usecase/integration_test.go @@ -0,0 +1,99 @@ +package usecase + +import ( + "context" + "path/filepath" + "testing" + + "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" + "gitea.maximumdirect.net/eric/scriptorium/internal/domain" + "gitea.maximumdirect.net/eric/scriptorium/internal/profile" + "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" + "gitea.maximumdirect.net/eric/scriptorium/internal/validate" +) + +type integrationLLM struct{} + +func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { + return &domain.GenerateResponse{ + Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`, + Usage: domain.TokenUsage{ + PromptTokens: 42, + CompletionTokens: 36, + TotalTokens: 78, + }, + }, nil +} + +func TestRunnerIntegrationWithProfilesFixturesAndValidation(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..")) + if err != nil { + t.Fatalf("failed to resolve repo root: %v", err) + } + + profilesDir := filepath.Join(root, "profiles") + schemasDir := filepath.Join(root, "schemas") + fixturesDir := filepath.Join(root, "examples", "fixtures") + + runner := NewRunner( + profile.NewFilesystemRepository(profilesDir), + artifact.NewCompositeReader(), + prompt.NewGoRenderer(), + &integrationLLM{}, + validate.NewStandardValidator(schemasDir), + ) + + res, err := runner.Run(context.Background(), domain.RunRequest{ + ProfileID: "generic.structured_events", + Inputs: map[string]domain.ArtifactRef{ + "transcript": { + Type: domain.ArtifactRefFile, + URI: filepath.Join(fixturesDir, "transcript.md"), + }, + "glossary": { + Type: domain.ArtifactRefFile, + URI: filepath.Join(fixturesDir, "glossary.yml"), + }, + }, + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + if res.ProfileID != "generic.structured_events" { + t.Fatalf("unexpected profile id: %q", res.ProfileID) + } + if res.ProfileVersion != "1.0.0" { + t.Fatalf("unexpected profile version: %q", res.ProfileVersion) + } + if res.Validation.Status != domain.ValidationPassed { + t.Fatalf("expected passed validation, got %q", res.Validation.Status) + } + if res.Validation.Mode != domain.ValidationJSONSchema { + t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode) + } + if res.Artifact.ContentType != "application/json" { + t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType) + } + if len(res.RawOutput) == 0 { + t.Fatal("expected raw output to be preserved") + } + if res.PromptHash == "" { + t.Fatal("expected non-empty prompt hash") + } + if len(res.InputHashes) != 2 { + t.Fatalf("expected two input hashes, got %d", len(res.InputHashes)) + } + if res.InputHashes["transcript"] == "" || res.InputHashes["glossary"] == "" { + t.Fatalf("expected both input hashes to be set, got %#v", res.InputHashes) + } + if res.Usage.TotalTokens != 78 { + t.Fatalf("expected usage from fake llm, got %+v", res.Usage) + } + if res.StartTime.IsZero() || res.EndTime.IsZero() { + t.Fatal("expected start/end timestamps") + } + if res.EndTime.Before(res.StartTime) { + t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) + } +} diff --git a/profiles/dnd.session_recap.yaml b/profiles/dnd.session_recap.yaml new file mode 100644 index 0000000..6211f35 --- /dev/null +++ b/profiles/dnd.session_recap.yaml @@ -0,0 +1,31 @@ +id: dnd.session_recap +version: "1.0.0" +description: Example D&D session recap profile for demonstration only. +expected_inputs: + - transcript + - glossary +templates: + - role: system + content: | + You create concise tabletop RPG session recaps. + Stay factual and avoid inventing events. + - role: user + content: | + Produce a recap with these sections: + - Session Highlights + - Open Threads + - NPC Mentions + + Transcript: + {{input "transcript"}} + + Glossary: + {{input "glossary"}} +model_defaults: + endpoint: http://localhost:8000/v1 + model: gpt-4o-mini + temperature: 0.3 + max_tokens: 900 +output_format: markdown +validation: + validation_mode: basic diff --git a/profiles/generic.markdown_summary.yaml b/profiles/generic.markdown_summary.yaml new file mode 100644 index 0000000..b094081 --- /dev/null +++ b/profiles/generic.markdown_summary.yaml @@ -0,0 +1,28 @@ +id: generic.markdown_summary +version: "1.0.0" +description: Generic markdown summary from transcript and glossary. +expected_inputs: + - transcript + - glossary +templates: + - role: system + content: | + You are a concise analysis assistant. + Write clear Markdown output. + - role: user + content: | + Summarize the following source transcript: + + {{input "transcript"}} + + Reference glossary: + + {{input "glossary"}} +model_defaults: + endpoint: http://localhost:8000/v1 + model: gpt-4o-mini + temperature: 0.2 + max_tokens: 700 +output_format: markdown +validation: + validation_mode: basic diff --git a/profiles/generic.structured_events.yaml b/profiles/generic.structured_events.yaml new file mode 100644 index 0000000..2722a49 --- /dev/null +++ b/profiles/generic.structured_events.yaml @@ -0,0 +1,31 @@ +id: generic.structured_events +version: "1.0.0" +description: Produce structured event JSON from a transcript. +expected_inputs: + - transcript + - glossary +templates: + - role: system + content: | + Return only JSON following the requested schema. + Do not include commentary. + - role: user + content: | + Extract important events from the transcript and emit JSON. + + Transcript: + {{input "transcript"}} + + Glossary: + {{input "glossary"}} +model_defaults: + endpoint: http://localhost:8000/v1 + model: gpt-4o-mini + temperature: 0.0 + max_tokens: 500 +output_format: json +validation: + format: json + validation_mode: json_schema + schema_path: structured_events.schema.json + repair_attempts: 0 diff --git a/schemas/structured_events.schema.json b/schemas/structured_events.schema.json new file mode 100644 index 0000000..45a026b --- /dev/null +++ b/schemas/structured_events.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.local/structured_events.schema.json", + "type": "object", + "required": ["summary", "events"], + "properties": { + "summary": { + "type": "string", + "minLength": 1 + }, + "events": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["title", "type"], + "properties": { + "title": {"type": "string", "minLength": 1}, + "type": {"type": "string", "enum": ["combat", "social", "travel", "discovery", "other"]}, + "notes": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +}