From 2bbf13e73943cd4e319a722ec7227d488761c31f Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 27 Jul 2026 21:59:53 +0000 Subject: [PATCH] Add framework contract test corpus --- engine_test.go | 116 ++++++++++++++---- testdata/framework/fixtures/glossary.yml | 2 + testdata/framework/fixtures/transcript.md | 2 + .../framework/profiles/contract-fast.yaml | 7 ++ .../framework/profiles/contract-quality.yaml | 7 ++ .../contract.markdown_summary.system.md | 1 + .../prompts/contract.markdown_summary.user.md | 7 ++ .../prompts/contract.markdown_summary.yaml | 20 +++ .../contract.structured_events.system.md | 1 + .../contract.structured_events.user.md | 7 ++ .../prompts/contract.structured_events.yaml | 21 ++++ .../schemas/structured_events.schema.json | 19 +++ 12 files changed, 187 insertions(+), 23 deletions(-) create mode 100644 testdata/framework/fixtures/glossary.yml create mode 100644 testdata/framework/fixtures/transcript.md create mode 100644 testdata/framework/profiles/contract-fast.yaml create mode 100644 testdata/framework/profiles/contract-quality.yaml create mode 100644 testdata/framework/prompts/contract.markdown_summary.system.md create mode 100644 testdata/framework/prompts/contract.markdown_summary.user.md create mode 100644 testdata/framework/prompts/contract.markdown_summary.yaml create mode 100644 testdata/framework/prompts/contract.structured_events.system.md create mode 100644 testdata/framework/prompts/contract.structured_events.user.md create mode 100644 testdata/framework/prompts/contract.structured_events.yaml create mode 100644 testdata/framework/schemas/structured_events.schema.json diff --git a/engine_test.go b/engine_test.go index 62add2d..412eac8 100644 --- a/engine_test.go +++ b/engine_test.go @@ -20,6 +20,21 @@ import ( "gitea.maximumdirect.net/eric/scriptorium" ) +const ( + frameworkContractRoot = "./testdata/framework" + frameworkPromptDir = frameworkContractRoot + "/prompts" + frameworkProfileDir = frameworkContractRoot + "/profiles" + frameworkSchemaDir = frameworkContractRoot + "/schemas" + + frameworkMarkdownSummaryPromptID = "contract.markdown_summary" + frameworkStructuredEventsPromptID = "contract.structured_events" + frameworkFastProfileID = "contract-fast" + frameworkQualityProfileID = "contract-quality" + + frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md" + frameworkGlossaryPath = frameworkContractRoot + "/fixtures/glossary.yml" +) + func TestNewEngineRejectsMissingPromptDir(t *testing.T) { _, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"}) if !errors.Is(err, scriptorium.ErrInvalidConfig) { @@ -34,33 +49,88 @@ func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { } } -func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) { - engine := newExampleEngine(t) - - prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File("./examples/fixtures/transcript.md"), - "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), - }, +func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { + engine, err := scriptorium.NewEngine(scriptorium.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, }) if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) + t.Fatalf("construct engine from framework contract corpus: %v", err) } - if prepared.PromptID != "generic.markdown_summary" { - t.Fatalf("unexpected prompt id: %q", prepared.PromptID) + + tests := []struct { + name string + promptID string + profileID string + model string + structured bool + }{ + { + name: "markdown summary", + promptID: frameworkMarkdownSummaryPromptID, + profileID: frameworkFastProfileID, + model: "contract-fast-model", + }, + { + name: "structured events", + promptID: frameworkStructuredEventsPromptID, + profileID: frameworkQualityProfileID, + model: "contract-quality-model", + structured: true, + }, } - if prepared.SelectedProfileID != "local-fast" { - t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) - } - if prepared.EffectiveModelParams.Model != "gpt-4o-mini" { - t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) - } - if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { - t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ + PromptID: tt.promptID, + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.File(frameworkTranscriptPath), + "glossary": scriptorium.File(frameworkGlossaryPath), + }, + }) + if err != nil { + t.Fatalf("prepare framework contract prompt: %v", err) + } + if prepared.PromptID != tt.promptID { + t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID) + } + if prepared.SelectedProfileID != tt.profileID { + t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID) + } + if prepared.EffectiveModelParams.Model != tt.model { + t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model) + } + if len(prepared.Messages) != 2 { + t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) + } + if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") { + t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content) + } + if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { + t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) + } + + if !tt.structured { + if prepared.StructuredOutput != nil { + t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput) + } + return + } + + if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { + t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput) + } + schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) + if !ok || schema["type"] != "object" { + t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema) + } + properties, ok := schema["properties"].(map[string]any) + if !ok || properties["events"] == nil { + t.Fatalf("expected loaded events schema property, got %#v", schema) + } + }) } } diff --git a/testdata/framework/fixtures/glossary.yml b/testdata/framework/fixtures/glossary.yml new file mode 100644 index 0000000..503c677 --- /dev/null +++ b/testdata/framework/fixtures/glossary.yml @@ -0,0 +1,2 @@ +archive: A catalogued collection of written records. +marker: A small label used to classify an entry. diff --git a/testdata/framework/fixtures/transcript.md b/testdata/framework/fixtures/transcript.md new file mode 100644 index 0000000..b74a773 --- /dev/null +++ b/testdata/framework/fixtures/transcript.md @@ -0,0 +1,2 @@ +Nia labels the archive. +The archive receives a blue marker. diff --git a/testdata/framework/profiles/contract-fast.yaml b/testdata/framework/profiles/contract-fast.yaml new file mode 100644 index 0000000..5ac326d --- /dev/null +++ b/testdata/framework/profiles/contract-fast.yaml @@ -0,0 +1,7 @@ +id: contract-fast +endpoint: http://localhost:8000/v1 +model: contract-fast-model +temperature: 0.2 +max_tokens: 500 +top_p: 1 +timeout_seconds: 90 diff --git a/testdata/framework/profiles/contract-quality.yaml b/testdata/framework/profiles/contract-quality.yaml new file mode 100644 index 0000000..4477c74 --- /dev/null +++ b/testdata/framework/profiles/contract-quality.yaml @@ -0,0 +1,7 @@ +id: contract-quality +endpoint: http://localhost:8000/v1 +model: contract-quality-model +temperature: 0.1 +max_tokens: 1000 +top_p: 0.9 +timeout_seconds: 120 diff --git a/testdata/framework/prompts/contract.markdown_summary.system.md b/testdata/framework/prompts/contract.markdown_summary.system.md new file mode 100644 index 0000000..3e649a2 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.system.md @@ -0,0 +1 @@ +You summarize synthetic archive notes in clear Markdown. diff --git a/testdata/framework/prompts/contract.markdown_summary.user.md b/testdata/framework/prompts/contract.markdown_summary.user.md new file mode 100644 index 0000000..aa47620 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.user.md @@ -0,0 +1,7 @@ +Summarize this transcript: + +{{input "transcript"}} + +Optional glossary: + +{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.markdown_summary.yaml b/testdata/framework/prompts/contract.markdown_summary.yaml new file mode 100644 index 0000000..3242ff5 --- /dev/null +++ b/testdata/framework/prompts/contract.markdown_summary.yaml @@ -0,0 +1,20 @@ +id: contract.markdown_summary +version: "1.0.0" +default_profile: contract-fast +description: Summarize a synthetic transcript in Markdown. +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml +messages: + - role: system + content_file: ./contract.markdown_summary.system.md + - role: user + content_file: ./contract.markdown_summary.user.md +output: + format: markdown + validation_mode: basic + repair_attempts: 0 diff --git a/testdata/framework/prompts/contract.structured_events.system.md b/testdata/framework/prompts/contract.structured_events.system.md new file mode 100644 index 0000000..eba5946 --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.system.md @@ -0,0 +1 @@ +Return only JSON that satisfies the requested event schema. diff --git a/testdata/framework/prompts/contract.structured_events.user.md b/testdata/framework/prompts/contract.structured_events.user.md new file mode 100644 index 0000000..9bcc0cc --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.user.md @@ -0,0 +1,7 @@ +Extract events from this transcript: + +{{input "transcript"}} + +Optional glossary: + +{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.structured_events.yaml b/testdata/framework/prompts/contract.structured_events.yaml new file mode 100644 index 0000000..607fad8 --- /dev/null +++ b/testdata/framework/prompts/contract.structured_events.yaml @@ -0,0 +1,21 @@ +id: contract.structured_events +version: "1.0.0" +default_profile: contract-quality +description: Extract synthetic events as structured JSON. +inputs: + - name: transcript + required: true + content_type: text/markdown + - name: glossary + required: false + content_type: text/yaml +messages: + - role: system + content_file: ./contract.structured_events.system.md + - role: user + content_file: ./contract.structured_events.user.md +output: + format: json + validation_mode: json_schema + schema_path: structured_events.schema.json + repair_attempts: 0 diff --git a/testdata/framework/schemas/structured_events.schema.json b/testdata/framework/schemas/structured_events.schema.json new file mode 100644 index 0000000..4791fe6 --- /dev/null +++ b/testdata/framework/schemas/structured_events.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["events"], + "properties": { + "events": { + "type": "array", + "items": { + "type": "object", + "required": ["title"], + "properties": { + "title": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +}