Add framework contract test corpus

engine_test.go

@@ -20,6 +20,21 @@ import (
 	"gitea.maximumdirect.net/eric/scriptorium"
 )
 
+const (
+	frameworkContractRoot = "./testdata/framework"
+	frameworkPromptDir    = frameworkContractRoot + "/prompts"
+	frameworkProfileDir   = frameworkContractRoot + "/profiles"
+	frameworkSchemaDir    = frameworkContractRoot + "/schemas"
+
+	frameworkMarkdownSummaryPromptID  = "contract.markdown_summary"
+	frameworkStructuredEventsPromptID = "contract.structured_events"
+	frameworkFastProfileID            = "contract-fast"
+	frameworkQualityProfileID         = "contract-quality"
+
+	frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md"
+	frameworkGlossaryPath   = frameworkContractRoot + "/fixtures/glossary.yml"
+)
+
 func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
 	_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
 	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
@@ -34,33 +49,88 @@ func TestNewEngineAcceptsMissingProfileDir(t *testing.T) {
 	}
 }
 
-func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
+func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) {
-	engine := newExampleEngine(t)
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
-
+		PromptDir:  frameworkPromptDir,
-	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
+		ProfileDir: frameworkProfileDir,
-		PromptID: "generic.markdown_summary",
+		SchemaDir:  frameworkSchemaDir,
-		Inputs: map[string]scriptorium.ArtifactRef{
-			"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
-			"glossary":   scriptorium.File("./examples/fixtures/glossary.yml"),
-		},
 	})
 	if err != nil {
-		t.Fatalf("expected prepare to succeed, got %v", err)
+		t.Fatalf("construct engine from framework contract corpus: %v", err)
 	}
-	if prepared.PromptID != "generic.markdown_summary" {
+
-		t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
+	tests := []struct {
+		name       string
+		promptID   string
+		profileID  string
+		model      string
+		structured bool
+	}{
+		{
+			name:      "markdown summary",
+			promptID:  frameworkMarkdownSummaryPromptID,
+			profileID: frameworkFastProfileID,
+			model:     "contract-fast-model",
+		},
+		{
+			name:       "structured events",
+			promptID:   frameworkStructuredEventsPromptID,
+			profileID:  frameworkQualityProfileID,
+			model:      "contract-quality-model",
+			structured: true,
+		},
 	}
-	if prepared.SelectedProfileID != "local-fast" {
+
-		t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
+	for _, tt := range tests {
-	}
+		t.Run(tt.name, func(t *testing.T) {
-	if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
+			prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
-		t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
+				PromptID: tt.promptID,
-	}
+				Inputs: map[string]scriptorium.ArtifactRef{
-	if len(prepared.Messages) != 2 {
+					"transcript": scriptorium.File(frameworkTranscriptPath),
-		t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
+					"glossary":   scriptorium.File(frameworkGlossaryPath),
-	}
+				},
-	if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
+			})
-		t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
+			if err != nil {
+				t.Fatalf("prepare framework contract prompt: %v", err)
+			}
+			if prepared.PromptID != tt.promptID {
+				t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID)
+			}
+			if prepared.SelectedProfileID != tt.profileID {
+				t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID)
+			}
+			if prepared.EffectiveModelParams.Model != tt.model {
+				t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model)
+			}
+			if len(prepared.Messages) != 2 {
+				t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
+			}
+			if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") {
+				t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content)
+			}
+			if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
+				t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
+			}
+
+			if !tt.structured {
+				if prepared.StructuredOutput != nil {
+					t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput)
+				}
+				return
+			}
+
+			if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
+				t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput)
+			}
+			schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)
+			if !ok || schema["type"] != "object" {
+				t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema)
+			}
+			properties, ok := schema["properties"].(map[string]any)
+			if !ok || properties["events"] == nil {
+				t.Fatalf("expected loaded events schema property, got %#v", schema)
+			}
+		})
 	}
 }
 

testdata/framework/fixtures/glossary.yml

@@ -0,0 +1,2 @@
+archive: A catalogued collection of written records.
+marker: A small label used to classify an entry.

testdata/framework/fixtures/transcript.md

@@ -0,0 +1,2 @@
+Nia labels the archive.
+The archive receives a blue marker.

testdata/framework/profiles/contract-fast.yaml

@@ -0,0 +1,7 @@
+id: contract-fast
+endpoint: http://localhost:8000/v1
+model: contract-fast-model
+temperature: 0.2
+max_tokens: 500
+top_p: 1
+timeout_seconds: 90

testdata/framework/profiles/contract-quality.yaml

@@ -0,0 +1,7 @@
+id: contract-quality
+endpoint: http://localhost:8000/v1
+model: contract-quality-model
+temperature: 0.1
+max_tokens: 1000
+top_p: 0.9
+timeout_seconds: 120

testdata/framework/prompts/contract.markdown_summary.system.md

@@ -0,0 +1 @@
+You summarize synthetic archive notes in clear Markdown.

testdata/framework/prompts/contract.markdown_summary.user.md

@@ -0,0 +1,7 @@
+Summarize this transcript:
+
+{{input "transcript"}}
+
+Optional glossary:
+
+{{input "glossary"}}

testdata/framework/prompts/contract.markdown_summary.yaml

@@ -0,0 +1,20 @@
+id: contract.markdown_summary
+version: "1.0.0"
+default_profile: contract-fast
+description: Summarize a synthetic transcript in Markdown.
+inputs:
+  - name: transcript
+    required: true
+    content_type: text/markdown
+  - name: glossary
+    required: false
+    content_type: text/yaml
+messages:
+  - role: system
+    content_file: ./contract.markdown_summary.system.md
+  - role: user
+    content_file: ./contract.markdown_summary.user.md
+output:
+  format: markdown
+  validation_mode: basic
+  repair_attempts: 0

testdata/framework/prompts/contract.structured_events.system.md

@@ -0,0 +1 @@
+Return only JSON that satisfies the requested event schema.

testdata/framework/prompts/contract.structured_events.user.md

@@ -0,0 +1,7 @@
+Extract events from this transcript:
+
+{{input "transcript"}}
+
+Optional glossary:
+
+{{input "glossary"}}

testdata/framework/prompts/contract.structured_events.yaml

@@ -0,0 +1,21 @@
+id: contract.structured_events
+version: "1.0.0"
+default_profile: contract-quality
+description: Extract synthetic events as structured JSON.
+inputs:
+  - name: transcript
+    required: true
+    content_type: text/markdown
+  - name: glossary
+    required: false
+    content_type: text/yaml
+messages:
+  - role: system
+    content_file: ./contract.structured_events.system.md
+  - role: user
+    content_file: ./contract.structured_events.user.md
+output:
+  format: json
+  validation_mode: json_schema
+  schema_path: structured_events.schema.json
+  repair_attempts: 0

testdata/framework/schemas/structured_events.schema.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "required": ["events"],
+  "properties": {
+    "events": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["title"],
+        "properties": {
+          "title": {"type": "string"}
+        },
+        "additionalProperties": false
+      }
+    }
+  },
+  "additionalProperties": false
+}