Add framework contract test corpus

engine_test.go

@@ -20,6 +20,21 @@ import (
 	"gitea.maximumdirect.net/eric/scriptorium"
 )
 
+const (
+	frameworkContractRoot = "./testdata/framework"
+	frameworkPromptDir    = frameworkContractRoot + "/prompts"
+	frameworkProfileDir   = frameworkContractRoot + "/profiles"
+	frameworkSchemaDir    = frameworkContractRoot + "/schemas"
+
+	frameworkMarkdownSummaryPromptID  = "contract.markdown_summary"
+	frameworkStructuredEventsPromptID = "contract.structured_events"
+	frameworkFastProfileID            = "contract-fast"
+	frameworkQualityProfileID         = "contract-quality"
+
+	frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md"
+	frameworkGlossaryPath   = frameworkContractRoot + "/fixtures/glossary.yml"
+)
+
 func TestNewEngineRejectsMissingPromptDir(t *testing.T) {
 	_, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"})
 	if !errors.Is(err, scriptorium.ErrInvalidConfig) {
@@ -34,33 +49,88 @@ func TestNewEngineAcceptsMissingProfileDir(t *testing.T) {
 	}
 }
 
-func TestPrepareWorksWithExampleDirectoriesAndFileInputs(t *testing.T) {
-	engine := newExampleEngine(t)
-
-	prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
-		PromptID: "generic.markdown_summary",
-		Inputs: map[string]scriptorium.ArtifactRef{
-			"transcript": scriptorium.File("./examples/fixtures/transcript.md"),
-			"glossary":   scriptorium.File("./examples/fixtures/glossary.yml"),
-		},
+func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) {
+	engine, err := scriptorium.NewEngine(scriptorium.Config{
+		PromptDir:  frameworkPromptDir,
+		ProfileDir: frameworkProfileDir,
+		SchemaDir:  frameworkSchemaDir,
 	})
 	if err != nil {
-		t.Fatalf("expected prepare to succeed, got %v", err)
+		t.Fatalf("construct engine from framework contract corpus: %v", err)
 	}
-	if prepared.PromptID != "generic.markdown_summary" {
-		t.Fatalf("unexpected prompt id: %q", prepared.PromptID)
+
+	tests := []struct {
+		name       string
+		promptID   string
+		profileID  string
+		model      string
+		structured bool
+	}{
+		{
+			name:      "markdown summary",
+			promptID:  frameworkMarkdownSummaryPromptID,
+			profileID: frameworkFastProfileID,
+			model:     "contract-fast-model",
+		},
+		{
+			name:       "structured events",
+			promptID:   frameworkStructuredEventsPromptID,
+			profileID:  frameworkQualityProfileID,
+			model:      "contract-quality-model",
+			structured: true,
+		},
 	}
-	if prepared.SelectedProfileID != "local-fast" {
-		t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
-	}
-	if prepared.EffectiveModelParams.Model != "gpt-4o-mini" {
-		t.Fatalf("unexpected effective model: %q", prepared.EffectiveModelParams.Model)
-	}
-	if len(prepared.Messages) != 2 {
-		t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
-	}
-	if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
-		t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{
+				PromptID: tt.promptID,
+				Inputs: map[string]scriptorium.ArtifactRef{
+					"transcript": scriptorium.File(frameworkTranscriptPath),
+					"glossary":   scriptorium.File(frameworkGlossaryPath),
+				},
+			})
+			if err != nil {
+				t.Fatalf("prepare framework contract prompt: %v", err)
+			}
+			if prepared.PromptID != tt.promptID {
+				t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID)
+			}
+			if prepared.SelectedProfileID != tt.profileID {
+				t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID)
+			}
+			if prepared.EffectiveModelParams.Model != tt.model {
+				t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model)
+			}
+			if len(prepared.Messages) != 2 {
+				t.Fatalf("expected rendered messages, got %d", len(prepared.Messages))
+			}
+			if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") {
+				t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content)
+			}
+			if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" {
+				t.Fatalf("expected input hashes, got %#v", prepared.InputHashes)
+			}
+
+			if !tt.structured {
+				if prepared.StructuredOutput != nil {
+					t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput)
+				}
+				return
+			}
+
+			if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil {
+				t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput)
+			}
+			schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)
+			if !ok || schema["type"] != "object" {
+				t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema)
+			}
+			properties, ok := schema["properties"].(map[string]any)
+			if !ok || properties["events"] == nil {
+				t.Fatalf("expected loaded events schema property, got %#v", schema)
+			}
+		})
 	}
 }