From c6c747e94d004417156c36290e49ae82d9fabf60 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 27 Jul 2026 22:04:08 +0000 Subject: [PATCH] Move framework tests to contract fixtures --- engine_test.go | 322 ++++++++++++++++----------- internal/usecase/integration_test.go | 127 ----------- 2 files changed, 192 insertions(+), 257 deletions(-) delete mode 100644 internal/usecase/integration_test.go diff --git a/engine_test.go b/engine_test.go index 412eac8..45bf9b3 100644 --- a/engine_test.go +++ b/engine_test.go @@ -36,14 +36,14 @@ const ( ) func TestNewEngineRejectsMissingPromptDir(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: "./examples/profiles"}) + _, err := scriptorium.NewEngine(scriptorium.Config{ProfileDir: frameworkProfileDir}) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}) + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}) if err != nil { t.Fatalf("expected missing profile dir to use built-ins, got %v", err) } @@ -135,10 +135,10 @@ func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { } func TestPrepareWorksWithInlineInputs(t *testing.T) { - engine := newExampleEngine(t) + engine := newContractEngine(t) prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin scouts the tower.\nKara lights a lantern."), "glossary": scriptorium.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"), @@ -161,12 +161,22 @@ func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { const secret = "public-api-test-secret" t.Setenv(envName, secret) - engine := newExampleEngine(t) + profileDir := t.TempDir() + writePublicProfileFileWithAPIKeyEnv(t, profileDir, "prepared-secret", "http://localhost:8000/v1", "prepared-secret-model", envName) + engine, err := scriptorium.NewEngine(scriptorium.Config{ + PromptDir: frameworkPromptDir, + ProfileDir: profileDir, + SchemaDir: frameworkSchemaDir, + }) + if err != nil { + t.Fatalf("expected engine construction to succeed, got %v", err) + } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.structured_events", + PromptID: frameworkStructuredEventsPromptID, + ProfileID: "prepared-secret", Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File("./examples/fixtures/transcript.md"), - "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), + "transcript": scriptorium.File(frameworkTranscriptPath), + "glossary": scriptorium.File(frameworkGlossaryPath), }, }) if err != nil { @@ -194,8 +204,8 @@ func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) { const secret = "run-request-secret" req := scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", - ProfileID: "local-fast", + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: frameworkFastProfileID, APIKey: secret, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -265,15 +275,15 @@ func TestGenerateRequestFormattingRedactsDirectAPIKey(t *testing.T) { } func TestPreparePreservesExplicitZeroExecutionOverrides(t *testing.T) { - engine := newExampleEngine(t) + engine := newContractEngine(t) zeroFloat := 0.0 zeroInt := 0 prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File("./examples/fixtures/transcript.md"), - "glossary": scriptorium.File("./examples/fixtures/glossary.yml"), + "transcript": scriptorium.File(frameworkTranscriptPath), + "glossary": scriptorium.File(frameworkGlossaryPath), }, Execution: &scriptorium.ExecutionTargetOverride{ Temperature: &zeroFloat, @@ -308,10 +318,10 @@ func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { }, }, } - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -338,7 +348,7 @@ func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid { t.Fatalf("expected passed validation, got %+v", result.Validation) } - if result.PromptID != "generic.markdown_summary" || result.SelectedProfileID != "local-fast" || result.ModelName != "gpt-4o-mini" { + if result.PromptID != frameworkMarkdownSummaryPromptID || result.SelectedProfileID != frameworkFastProfileID || result.ModelName != "contract-fast-model" { t.Fatalf("unexpected run metadata: %+v", result) } if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { @@ -354,17 +364,69 @@ func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { } } +func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) { + fake := &fakeLLMClient{ + response: &scriptorium.GenerateResponse{ + Content: `{"events":[{"title":"Archive labelled"}]}`, + Usage: scriptorium.TokenUsage{ + PromptTokens: 42, + CompletionTokens: 36, + TotalTokens: 78, + }, + }, + } + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) + + result, err := engine.Run(context.Background(), scriptorium.RunRequest{ + PromptID: frameworkStructuredEventsPromptID, + Inputs: map[string]scriptorium.ArtifactRef{ + "transcript": scriptorium.File(frameworkTranscriptPath), + "glossary": scriptorium.File(frameworkGlossaryPath), + }, + }) + if err != nil { + t.Fatalf("expected run to succeed, got %v", err) + } + if result.PromptID != frameworkStructuredEventsPromptID || result.SelectedProfileID != frameworkQualityProfileID { + t.Fatalf("unexpected run metadata: %+v", result) + } + if result.RunID == "" || result.PromptHash == "" || result.RenderedPromptHash == "" { + t.Fatalf("expected run and prompt hashes, got %+v", result) + } + if result.InputHashes["transcript"] == "" || result.InputHashes["glossary"] == "" { + t.Fatalf("expected both input hashes, got %#v", result.InputHashes) + } + if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil || + fake.requests[0].StructuredOutput.Type != scriptorium.StructuredOutputJSONSchema || + fake.requests[0].StructuredOutput.JSONSchema == nil || + fake.requests[0].StructuredOutput.JSONSchema.Schema == nil { + t.Fatalf("expected provider JSON Schema structured output, got %+v", fake.requests) + } + if result.Validation.Status != scriptorium.ValidationPassed || !result.Validation.IsValid || result.Validation.Mode != scriptorium.ValidationJSONSchema { + t.Fatalf("expected passed JSON Schema validation, got %+v", result.Validation) + } + if result.Artifact.ContentType != "application/json" { + t.Fatalf("expected JSON artifact, got %q", result.Artifact.ContentType) + } + if result.RawOutput != fake.response.Content || result.Usage != fake.response.Usage { + t.Fatalf("expected preserved output and usage, got output=%q usage=%+v", result.RawOutput, result.Usage) + } + if result.StartTime.IsZero() || result.EndTime.IsZero() || result.EndTime.Before(result.StartTime) || result.Duration < 0 { + t.Fatalf("expected ordered non-zero timestamps and non-negative duration, got start=%v end=%v duration=%v", result.StartTime, result.EndTime, result.Duration) + } +} + func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { const directKey = "direct-injected-key" fake := &fakeLLMClient{ response: &scriptorium.GenerateResponse{Content: "ok"}, } - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) zeroFloat := 0.0 zeroInt := 0 _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, APIKey: directKey, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -387,7 +449,7 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) } - if req.Target.Model != "gpt-4o-mini" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 { + if req.Target.Model != "contract-fast-model" || req.Target.Temperature != 0 || req.Target.MaxTokens != 0 || req.Target.TopP != 0 || req.Target.TimeoutSeconds != 0 { t.Fatalf("unexpected effective target: %+v", req.Target) } if !req.TargetPresence.Temperature || !req.TargetPresence.MaxTokens || !req.TargetPresence.TopP || !req.TargetPresence.TimeoutSeconds { @@ -430,16 +492,16 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv) engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "direct-auth", APIKey: directKey, Inputs: map[string]scriptorium.ArtifactRef{ @@ -475,16 +537,16 @@ func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv) engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } baseReq := scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "direct-prepare", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -527,16 +589,16 @@ func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) { profileDir := t.TempDir() writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv) engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "requires-auth", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -555,10 +617,10 @@ func TestRunValidationFailureReturnsResult(t *testing.T) { fake := &fakeLLMClient{ response: &scriptorium.GenerateResponse{Content: ""}, } - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) result, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -600,7 +662,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { { name: "profile not found", req: scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "missing-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -612,9 +674,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { { name: "artifact load", req: scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.File("./examples/fixtures/does-not-exist.md"), + "transcript": scriptorium.File(filepath.Join(t.TempDir(), "does-not-exist.md")), }, }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, @@ -623,7 +685,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { { name: "prompt render", req: scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, }, client: &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}}, want: scriptorium.ErrPromptRender, @@ -631,7 +693,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { { name: "llm failure", req: scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -643,7 +705,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { { name: "validation runtime failure", req: scriptorium.RunRequest{ - PromptID: "generic.structured_events", + PromptID: frameworkStructuredEventsPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), }, @@ -659,9 +721,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) { t.Run(tc.name, func(t *testing.T) { schemaDir := tc.schemaDir if schemaDir == "" { - schemaDir = "./examples/schemas" + schemaDir = frameworkSchemaDir } - engine := newExampleEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client)) + engine := newContractEngineWithOptions(t, schemaDir, scriptorium.WithLLMClient(tc.client)) _, err := engine.Run(context.Background(), tc.req) if !errors.Is(err, tc.want) { t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) @@ -682,16 +744,16 @@ api_key: secret } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "raw-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -716,16 +778,16 @@ unknown_field: true } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "broken-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -744,15 +806,15 @@ func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) { missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts") engine, err := scriptorium.NewEngine(scriptorium.Config{ PromptDir: missingPromptDir, - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -769,17 +831,17 @@ func TestPromptRepositoryReadFailureMapsToPromptLoad(t *testing.T) { func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) { missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles") engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: missingProfileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", - ProfileID: "local-fast", + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: frameworkFastProfileID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -796,15 +858,15 @@ func TestSelectedProfileRepositoryReadFailureMapsToProfileLoad(t *testing.T) { func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { t.Setenv("OPENROUTER_API_KEY", "test-key") engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -852,16 +914,16 @@ func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -889,16 +951,16 @@ unexpected: true } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", + PromptDir: frameworkPromptDir, ProfileDir: profileDir, - SchemaDir: "./examples/schemas", + SchemaDir: frameworkSchemaDir, }) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -915,7 +977,7 @@ func TestPrepareWorksWithPromptFSAndRelativeContentFile(t *testing.T) { "assets/prompts/fs-summary.yaml": &fstest.MapFile{Data: []byte(` id: fs.summary version: "1.0.0" -default_profile: local-fast +default_profile: contract-fast inputs: - name: transcript required: true @@ -932,8 +994,8 @@ output: engine, err := scriptorium.NewEngine(scriptorium.Config{ PromptDir: t.TempDir(), - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithPromptFS(promptFS, "assets/prompts")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) @@ -958,7 +1020,7 @@ func TestPrepareWithPromptFSRejectsEscapedContentFile(t *testing.T) { "assets/prompts/fs-escape.yaml": &fstest.MapFile{Data: []byte(` id: fs.escape version: "1.0.0" -default_profile: local-fast +default_profile: contract-fast messages: - role: user content_file: ../outside.tmpl @@ -972,8 +1034,8 @@ output: engine, err := scriptorium.NewEngine(scriptorium.Config{ PromptDir: t.TempDir(), - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithPromptFS(promptFS, "assets/prompts")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) @@ -991,7 +1053,7 @@ func TestPrepareWorksWithPromptFile(t *testing.T) { if err := os.WriteFile(promptPath, []byte(` id: single.file.prompt version: "1.0.0" -default_profile: local-fast +default_profile: contract-fast inputs: - name: transcript required: true @@ -1010,8 +1072,8 @@ output: } engine, err := scriptorium.NewEngine(scriptorium.Config{ - ProfileDir: "./examples/profiles", - SchemaDir: "./examples/schemas", + ProfileDir: frameworkProfileDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithPromptFile(promptPath)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) @@ -1044,15 +1106,15 @@ model: profile-fs-model } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfileFS(profileFS, "profiles")) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1079,15 +1141,15 @@ model: profile-file-model } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfileFile(profilePath)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1104,8 +1166,8 @@ model: profile-file-model func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(scriptorium.Profile{ ID: "memory-profile", Endpoint: "http://memory-profile/v1", @@ -1116,7 +1178,7 @@ func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "memory-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1141,8 +1203,8 @@ model: profile-fs-model } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfileFS(profileFS, "profiles"), scriptorium.WithProfiles(scriptorium.Profile{ @@ -1156,7 +1218,7 @@ model: profile-fs-model } prepared, err := engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "mistral-small-3", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1172,7 +1234,7 @@ model: profile-fs-model } func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, scriptorium.WithProfiles( scriptorium.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, scriptorium.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, @@ -1196,15 +1258,15 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { }) engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "template-profile", APIKey: "template-key", Inputs: map[string]scriptorium.ArtifactRef{ @@ -1313,8 +1375,8 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { Transport: transport, } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, Timeout: tc.configTimeout, HTTPClient: httpClient, }, scriptorium.WithProfiles(scriptorium.Profile{ @@ -1335,7 +1397,7 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { defer cancel() _, err = engine.Run(ctx, scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "layered-timeout", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1376,7 +1438,7 @@ func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { ExtraParams: cyclic, }) - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, scriptorium.WithProfiles(prof), ) if !errors.Is(err, scriptorium.ErrInvalidConfig) { @@ -1402,8 +1464,8 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi extraParams["added"] = "mutated-after-construction" engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(prof), scriptorium.WithLLMClient(fake)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) @@ -1411,7 +1473,7 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi nested["added"] = "mutated-after-construction" _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "nested-template-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1434,8 +1496,8 @@ func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testi func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(scriptorium.Profile{ ID: "requires-key", Endpoint: "http://requires-key/v1", @@ -1447,7 +1509,7 @@ func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { } req := scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "requires-key", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1466,8 +1528,8 @@ func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(scriptorium.Profile{ ID: "no-key-required", Endpoint: "http://no-key/v1", @@ -1478,7 +1540,7 @@ func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { } _, err = engine.Prepare(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "no-key-required", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1500,8 +1562,8 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, scriptorium.WithProfiles(scriptorium.Profile{ ID: "copy-profile", @@ -1519,7 +1581,7 @@ func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { extraParams["added"] = "mutated" _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "copy-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1553,7 +1615,7 @@ func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, scriptorium.WithProfiles(scriptorium.Profile{ ID: "invalid-extra-params", Endpoint: "http://invalid/v1", @@ -1583,7 +1645,7 @@ func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, scriptorium.WithProfiles(scriptorium.Profile{ ID: "cyclic-extra-params", Endpoint: "http://cyclic/v1", @@ -1602,7 +1664,7 @@ func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}} engine, err := scriptorium.NewEngine(scriptorium.Config{ PromptDir: t.TempDir(), - ProfileDir: "./examples/profiles", + ProfileDir: frameworkProfileDir, SchemaDir: t.TempDir(), }, scriptorium.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), @@ -1639,7 +1701,7 @@ func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: `{"events":[]}`}} engine, err := scriptorium.NewEngine(scriptorium.Config{ - ProfileDir: "./examples/profiles", + ProfileDir: frameworkProfileDir, }, scriptorium.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), scriptorium.WithSchemaFile(schemaPath), @@ -1686,7 +1748,7 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: "./examples/prompts"}, tc.opt) + _, err := scriptorium.NewEngine(scriptorium.Config{PromptDir: frameworkPromptDir}, tc.opt) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } @@ -1707,15 +1769,15 @@ func TestPackageOptionsComposeFromSlice(t *testing.T) { } engine, err := scriptorium.NewEngine(scriptorium.Config{ - PromptDir: "./examples/prompts", - SchemaDir: "./examples/schemas", + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, }, options...) if err != nil { t.Fatalf("expected package-provided options to compose, got %v", err) } _, err = engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, ProfileID: "slice-profile", Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), @@ -1735,7 +1797,7 @@ func TestPackageOptionsComposeFromSlice(t *testing.T) { func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) labels := map[string]string{"route": "primary"} counts := map[string]int{"retry_budget": 2} @@ -1754,7 +1816,7 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) } _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -1807,10 +1869,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -1843,10 +1905,10 @@ func TestRunRejectsCyclicExtraParams(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fake := &fakeLLMClient{response: &scriptorium.GenerateResponse{Content: "ok"}} - engine := newExampleEngineWithOptions(t, "./examples/schemas", scriptorium.WithLLMClient(fake)) + engine := newContractEngineWithOptions(t, frameworkSchemaDir, scriptorium.WithLLMClient(fake)) _, err := engine.Run(context.Background(), scriptorium.RunRequest{ - PromptID: "generic.markdown_summary", + PromptID: frameworkMarkdownSummaryPromptID, Inputs: map[string]scriptorium.ArtifactRef{ "transcript": scriptorium.Inline("Rin opens the gate."), "glossary": scriptorium.Inline("gate: A guarded passage."), @@ -1864,52 +1926,52 @@ func TestRunRejectsCyclicExtraParams(t *testing.T) { } func TestWithLLMClientRejectsNilClient(t *testing.T) { - _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas"), scriptorium.WithLLMClient(nil)) + _, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir), scriptorium.WithLLMClient(nil)) if !errors.Is(err, scriptorium.ErrInvalidConfig) { t.Fatalf("expected ErrInvalidConfig, got %v", err) } } func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { - if _, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")); err != nil { + if _, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) } } -func newExampleEngine(t *testing.T) *scriptorium.Engine { +func newContractEngine(t *testing.T) *scriptorium.Engine { t.Helper() for _, path := range []string{ - "./examples/prompts", - "./examples/profiles", - "./examples/schemas", + frameworkPromptDir, + frameworkProfileDir, + frameworkSchemaDir, } { if _, err := os.Stat(path); err != nil { - t.Fatalf("expected example path %s to exist: %v", path, err) + t.Fatalf("expected framework contract path %s to exist: %v", path, err) } } - engine, err := scriptorium.NewEngine(exampleConfig("./examples/schemas")) + engine, err := scriptorium.NewEngine(contractConfig(frameworkSchemaDir)) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } -func newExampleEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine { +func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...scriptorium.Option) *scriptorium.Engine { t.Helper() - engine, err := scriptorium.NewEngine(exampleConfig(schemaDir), opts...) + engine, err := scriptorium.NewEngine(contractConfig(schemaDir), opts...) if err != nil { t.Fatalf("expected engine construction to succeed, got %v", err) } return engine } -func exampleConfig(schemaDir string) scriptorium.Config { +func contractConfig(schemaDir string) scriptorium.Config { return scriptorium.Config{ - PromptDir: "./examples/prompts", - ProfileDir: "./examples/profiles", + PromptDir: frameworkPromptDir, + ProfileDir: frameworkProfileDir, SchemaDir: schemaDir, } } @@ -1962,7 +2024,7 @@ func publicStructuredPromptFS(id string, schemaPath string) fstest.MapFS { return fstest.MapFS{ "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`id: ` + id + ` version: "1.0.0" -default_profile: local-fast +default_profile: contract-fast inputs: - name: transcript required: true diff --git a/internal/usecase/integration_test.go b/internal/usecase/integration_test.go deleted file mode 100644 index f500218..0000000 --- a/internal/usecase/integration_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package usecase - -import ( - "context" - "path/filepath" - "testing" - - "gitea.maximumdirect.net/eric/scriptorium/internal/artifact" - "gitea.maximumdirect.net/eric/scriptorium/internal/domain" - "gitea.maximumdirect.net/eric/scriptorium/internal/profile" - "gitea.maximumdirect.net/eric/scriptorium/internal/prompt" - "gitea.maximumdirect.net/eric/scriptorium/internal/promptdef" - "gitea.maximumdirect.net/eric/scriptorium/internal/validate" -) - -type integrationLLM struct{} - -func (f *integrationLLM) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) { - lastIntegrationRequest = req - return &domain.GenerateResponse{ - Content: `{"summary":"Party discovered a captive scout beneath the tower.","events":[{"title":"Scout found in cellar","type":"discovery","notes":"Scout requested rescue from goblin raiders."}]}`, - Usage: domain.TokenUsage{ - PromptTokens: 42, - CompletionTokens: 36, - TotalTokens: 78, - }, - }, nil -} - -var lastIntegrationRequest domain.GenerateRequest - -func TestRunnerIntegrationWithPromptAndProfileFixturesAndValidation(t *testing.T) { - root, err := filepath.Abs(filepath.Join("..", "..")) - if err != nil { - t.Fatalf("failed to resolve repo root: %v", err) - } - - promptsDir := filepath.Join(root, "examples", "prompts") - profilesDir := filepath.Join(root, "examples", "profiles") - schemasDir := filepath.Join(root, "examples", "schemas") - fixturesDir := filepath.Join(root, "examples", "fixtures") - t.Setenv("SCRIPTORIUM_API_KEY", "test-key") - - runner := NewRunner( - promptdef.NewFilesystemRepository(promptsDir), - profile.NewFilesystemRepository(profilesDir), - artifact.NewCompositeReader(), - prompt.NewGoRenderer(), - &integrationLLM{}, - validate.NewStandardValidator(schemasDir), - ) - - res, err := runner.Run(context.Background(), domain.RunRequest{ - PromptID: "generic.structured_events", - Inputs: map[string]domain.ArtifactRef{ - "transcript": { - Type: domain.ArtifactRefFile, - URI: filepath.Join(fixturesDir, "transcript.md"), - }, - "glossary": { - Type: domain.ArtifactRefFile, - URI: filepath.Join(fixturesDir, "glossary.yml"), - }, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - - if res.PromptID != "generic.structured_events" { - t.Fatalf("unexpected prompt id: %q", res.PromptID) - } - if res.SelectedProfileID != "local-quality" { - t.Fatalf("expected selected profile local-quality from prompt default, got %q", res.SelectedProfileID) - } - if res.RunID == "" { - t.Fatal("expected run id") - } - if res.PromptHash == "" { - t.Fatal("expected prompt hash") - } - if res.PromptVersion != "1.0.0" { - t.Fatalf("unexpected prompt version: %q", res.PromptVersion) - } - if res.Validation.Status != domain.ValidationPassed { - t.Fatalf("expected passed validation, got %q", res.Validation.Status) - } - if res.Validation.Mode != domain.ValidationJSONSchema { - t.Fatalf("expected json_schema mode, got %q", res.Validation.Mode) - } - if lastIntegrationRequest.StructuredOutput == nil { - t.Fatal("expected provider-level structured output request for json_schema prompt") - } - if lastIntegrationRequest.StructuredOutput.Type != domain.StructuredOutputJSONSchema { - t.Fatalf("expected structured output type json_schema, got %q", lastIntegrationRequest.StructuredOutput.Type) - } - if lastIntegrationRequest.StructuredOutput.JSONSchema == nil || lastIntegrationRequest.StructuredOutput.JSONSchema.Schema == nil { - t.Fatalf("expected structured output json_schema payload, got %+v", lastIntegrationRequest.StructuredOutput.JSONSchema) - } - if res.Artifact.ContentType != "application/json" { - t.Fatalf("expected application/json output, got %q", res.Artifact.ContentType) - } - if len(res.RawOutput) == 0 { - t.Fatal("expected raw output to be preserved") - } - if res.PromptHash == "" { - t.Fatal("expected non-empty prompt hash") - } - if len(res.InputHashes) != 2 { - t.Fatalf("expected two input hashes, got %d", len(res.InputHashes)) - } - if res.InputHashes["transcript"] == "" || res.InputHashes["glossary"] == "" { - t.Fatalf("expected both input hashes to be set, got %#v", res.InputHashes) - } - if res.Usage.TotalTokens != 78 { - t.Fatalf("expected usage from fake llm, got %+v", res.Usage) - } - if res.StartTime.IsZero() || res.EndTime.IsZero() { - t.Fatal("expected start/end timestamps") - } - if res.EndTime.Before(res.StartTime) { - t.Fatalf("expected end >= start, got start=%v end=%v", res.StartTime, res.EndTime) - } - if res.Duration < 0 { - t.Fatalf("expected non-negative duration, got %s", res.Duration) - } -}