diff --git a/internal/framework/contracts/contracts_test.go b/internal/framework/contracts/contracts_test.go index bc6d012..30cf216 100644 --- a/internal/framework/contracts/contracts_test.go +++ b/internal/framework/contracts/contracts_test.go @@ -6,7 +6,6 @@ import ( "reflect" "testing" - "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/source" ) @@ -18,155 +17,6 @@ var _ Normalizer[fakeArtifact] = fakeNormalizer{} var _ StructuredLLMClient = fakeLLMClient{} var _ OutputEncoder = fakeOutputEncoder{} -func TestFakeExtractorReturnsTypedOutput(t *testing.T) { - extractor := fakeExtractor{ - key: "generic-extractor", - } - doc := &source.SourceDocument{ - ID: "source-1", - Kind: "document", - Format: "text/plain", - Digest: "sha256:abc123", - Units: []source.SourceUnit{ - {ID: 1, Kind: "section", Text: "Source text."}, - }, - } - - result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc}) - if err != nil { - t.Fatalf("Extract() error = %v, want nil", err) - } - - if extractor.Key() != "generic-extractor" { - t.Fatalf("Key() = %q, want generic-extractor", extractor.Key()) - } - if result.Value.Value != "example" { - t.Fatalf("Value = %q, want example", result.Value.Value) - } -} - -func TestFakeChunkerReturnsSourcePlan(t *testing.T) { - doc := &source.SourceDocument{ - ID: "source-1", - Kind: "document", - Format: "text/plain", - Digest: "sha256:abc123", - Units: []source.SourceUnit{ - {ID: 1, Kind: "section", Text: "Source text."}, - }, - } - chunker := fakeChunker{key: "generic-chunker"} - - result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc}) - if err != nil { - t.Fatalf("Plan() error = %v, want nil", err) - } - - if chunker.Key() != "generic-chunker" { - t.Fatalf("Key() = %q, want generic-chunker", chunker.Key()) - } - if len(result.Plan.Ranges) != 1 { - t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges)) - } - if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 { - t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan) - } -} - -func TestFakeChunkerReceivesPerRunContext(t *testing.T) { - doc := &source.SourceDocument{ - ID: "source-1", - Kind: "document", - Format: "text/plain", - Digest: "sha256:abc123", - Units: []source.SourceUnit{ - {ID: 1, Kind: "section", Text: "Source text."}, - }, - } - chunker := &recordingChunker{key: "llm-chunker"} - - if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil { - t.Fatalf("Plan() error = %v, want nil", err) - } - if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" { - t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request) - } -} - -func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) { - extractor := fakeExtractor{key: "generic-extractor"} - doc := &source.SourceDocument{ - ID: "source-1", - Kind: "document", - Format: "text/plain", - Digest: "sha256:abc123", - Units: []source.SourceUnit{ - {ID: 1, Kind: "section", Text: "First source text."}, - {ID: 2, Kind: "section", Text: "Second source text."}, - }, - } - chunk := source.Chunk{ - ID: "source-1:chunk:1", - SourceID: doc.ID, - Index: 1, - Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 2}, - Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`), - MediaType: "application/json", - Units: []source.SourceUnit{doc.Units[1]}, - } - - result, err := extractor.Extract(context.Background(), TypedExtractionRequest{ - Source: doc, - Chunk: &chunk, - AmbientContext: map[string]any{"mode": "chunked"}, - }) - if err != nil { - t.Fatalf("Extract() error = %v, want nil", err) - } - if result.Value.Value != "chunked" { - t.Fatalf("Value = %q, want chunked", result.Value.Value) - } -} - -func TestReferenceSetDataTypes(t *testing.T) { - references := ReferenceSet{ - Slots: map[string]ResolvedReferenceSlot{ - "roster": { - Slot: ReferenceSlot{ - Name: "roster", - Description: "Known characters", - Required: true, - AcceptedMediaTypes: []string{"text/plain"}, - Multiple: true, - MaxBytes: 4096, - }, - Items: []ReferenceItem{ - { - SlotName: "roster", - MediaType: "text/plain", - Content: []byte("Aria\nBryn\n"), - Digest: "sha256:reference", - Origin: ReferenceOrigin{ - Type: "file", - URI: "file:///tmp/roster.txt", - }, - SizeBytes: 10, - BindingSource: ReferenceBindingSourceConfig, - }, - }, - }, - }, - } - - item := references.Slots["roster"].Items[0] - if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" { - t.Fatalf("reference item = %#v, want constructed item fields", item) - } - if item.BindingSource != ReferenceBindingSourceConfig { - t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig) - } -} - func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) { if got := CloneReferenceSlots(nil); got != nil { t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got) @@ -319,61 +169,6 @@ func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) { } } -func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) { - extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}} - merger := fakeMerger{key: "generic-merger"} - normalizer := fakeNormalizer{key: "generic-normalizer"} - encoder := fakeOutputEncoder{key: "generic-output"} - - merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{ - LaneID: "generic-lane", - ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput}, - }) - if err != nil { - t.Fatalf("Merge() error = %v, want nil", err) - } - if merger.Key() != "generic-merger" { - t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key()) - } - if merged.Value.Value != "example" { - t.Fatalf("merged value = %q, want example", merged.Value.Value) - } - - normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{ - LaneID: "generic-lane", - MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value}, - }) - if err != nil { - t.Fatalf("Normalize() error = %v, want nil", err) - } - if normalizer.Key() != "generic-normalizer" { - t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key()) - } - if normalized.Value.Value != "example" { - t.Fatalf("normalized value = %q, want example", normalized.Value.Value) - } - - encoded, err := encoder.Encode(context.Background(), OutputRequest{ - Manifest: artifacts.RunManifest{RunID: "run-1"}, - NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}}, - }) - if err != nil { - t.Fatalf("Encode() error = %v, want nil", err) - } - if encoder.Key() != "generic-output" { - t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key()) - } - if len(encoded.Files) != 1 { - t.Fatalf("len(Files) = %d, want 1", len(encoded.Files)) - } - if encoded.Files[0].ContentType != "application/json" { - t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType) - } - if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` { - t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes) - } -} - func TestOutputFileJSONShapeOmitsBytes(t *testing.T) { file := OutputFile{ Name: "artifacts/events.json", @@ -441,24 +236,6 @@ func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPla }, nil } -type recordingChunker struct { - key string - request ChunkRequest -} - -func (chunker *recordingChunker) Key() string { - return chunker.key -} - -func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot { - return nil -} - -func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) { - chunker.request = req - return fakeChunker{key: chunker.key}.Plan(ctx, req) -} - type fakeExtractor struct { key string } diff --git a/internal/framework/llm/scriptorium_api_test.go b/internal/framework/llm/scriptorium_api_test.go deleted file mode 100644 index 66fae15..0000000 --- a/internal/framework/llm/scriptorium_api_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package llm - -import ( - "context" - "errors" - "testing" - "testing/fstest" - "time" - - "gitea.maximumdirect.net/eric/scriptorium" -) - -func TestScriptoriumPublicAPIGrounding(t *testing.T) { - // Keep this compile-time grounding close to the future Notarius adapter so - // dependency upgrades reveal API drift before the runtime cutover. - engine, err := scriptorium.NewEngine( - scriptorium.Config{ - PromptDir: "unused-when-prompt-option-is-set", - ProfileDir: "", - SchemaDir: "", - Timeout: time.Second, - }, - scriptorium.WithPromptFS(fstest.MapFS{}, "."), - scriptorium.WithProfileFS(fstest.MapFS{}, "."), - scriptorium.WithSchemaFS(fstest.MapFS{}, "."), - scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{ - ID: "test-profile", - Endpoint: "http://127.0.0.1:1/v1", - Model: "test-model", - APIKeyRequired: true, - ExtraParams: map[string]any{"mode": "test"}, - })), - scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}), - ) - if err != nil { - t.Fatalf("NewEngine() error = %v, want nil", err) - } - if engine == nil { - t.Fatalf("NewEngine() = nil, want engine") - } - - var ( - _ func(string) scriptorium.Option = scriptorium.WithPromptFile - _ func(string) scriptorium.Option = scriptorium.WithProfileFile - _ func(string) scriptorium.Option = scriptorium.WithSchemaFile - ) - - req := scriptorium.RunRequest{ - PromptID: "dnd.spells", - PromptVersion: "v1", - ProfileID: "test-profile", - APIKey: "request-scoped-secret", - Inputs: map[string]scriptorium.ArtifactRef{ - "transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`), - "glossary": scriptorium.Inline(""), - "roster": scriptorium.File("/tmp/roster.txt"), - }, - Vars: map[string]string{ - "session_id": "session-1", - }, - Execution: &scriptorium.ExecutionTargetOverride{ - Model: "override-model", - Temperature: ptr(0.2), - MaxTokens: ptr(100), - TopP: ptr(0.9), - TimeoutSeconds: ptr(30), - ServiceTier: "standard", - ReasoningEffort: "low", - APIKeyEnv: "SCRIPTORIUM_API_KEY", - ExtraParams: map[string]any{"provider_option": "value"}, - }, - Validation: &scriptorium.OutputContract{ - Format: scriptorium.FormatJSON, - ValidationMode: scriptorium.ValidationJSONSchema, - SchemaPath: "schemas/dnd_spells.v1.json", - RepairAttempts: 1, - }, - Metadata: map[string]string{ - "artifact_kind": "dnd_spell", - }, - } - if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline { - t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline) - } - if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile { - t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile) - } - - result := scriptorium.RunResult{ - RunID: "run-1", - Artifact: scriptorium.Artifact{ - Name: "output", - ContentType: "application/json", - Body: []byte(`{"ok":true}`), - URI: "inline://output", - Size: int64(len(`{"ok":true}`)), - Hash: "sha256:abc", - }, - RawOutput: `{"ok":true}`, - PromptID: req.PromptID, - PromptVersion: req.PromptVersion, - PromptHash: "prompt-hash", - RenderedPromptHash: "rendered-prompt-hash", - SelectedProfileID: req.ProfileID, - ModelName: "test-model", - Endpoint: "http://127.0.0.1:1/v1", - EffectiveModelParams: scriptorium.ExecutionTarget{ - Model: "test-model", - APIKeyEnv: "SCRIPTORIUM_API_KEY", - ExtraParams: map[string]any{"provider_option": "value"}, - ReasoningEffort: "low", - }, - InputHashes: map[string]string{ - "transcript": "sha256:def", - }, - Validation: scriptorium.ValidationResult{ - Status: scriptorium.ValidationPassed, - Mode: scriptorium.ValidationJSONSchema, - SchemaPath: req.Validation.SchemaPath, - RepairAttempts: 1, - IsValid: true, - }, - Usage: scriptorium.TokenUsage{ - PromptTokens: 10, - CompletionTokens: 5, - TotalTokens: 15, - CachedTokens: 3, - CacheWriteTokens: 2, - }, - StartTime: time.Unix(1, 0), - EndTime: time.Unix(2, 0), - Duration: time.Second, - } - if result.Validation.Status != scriptorium.ValidationPassed { - t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed) - } - if result.Usage.TotalTokens != 15 { - t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens) - } - - publicErrors := []error{ - scriptorium.ErrInvalidConfig, - scriptorium.ErrInvalidRequest, - scriptorium.ErrPromptNotFound, - scriptorium.ErrProfileNotFound, - scriptorium.ErrPromptLoad, - scriptorium.ErrProfileLoad, - scriptorium.ErrArtifactLoad, - scriptorium.ErrPromptRender, - scriptorium.ErrLLMGenerate, - scriptorium.ErrValidation, - } - for _, publicErr := range publicErrors { - if !errors.Is(publicErr, publicErr) { - t.Fatalf("sentinel error does not match itself: %v", publicErr) - } - } -} - -type scriptoriumGroundingLLMClient struct{} - -func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { - return &scriptorium.GenerateResponse{ - Content: `{"ok":true}`, - Usage: scriptorium.TokenUsage{ - PromptTokens: 1, - CompletionTokens: 1, - TotalTokens: 2, - }, - }, nil -} - -func ptr[T any](v T) *T { - return &v -} diff --git a/internal/framework/validate/validate_test.go b/internal/framework/validate/validate_test.go index c39df16..f671261 100644 --- a/internal/framework/validate/validate_test.go +++ b/internal/framework/validate/validate_test.go @@ -2,8 +2,6 @@ package validate import ( "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func TestApproved(t *testing.T) { @@ -33,8 +31,3 @@ func TestRejectedTrimsReasonAndMessage(t *testing.T) { t.Fatalf("Message = %q, want message", result.Message) } } - -func TestHelpersReturnValidationResults(t *testing.T) { - var _ contracts.ValidationResult = Approved() - var _ contracts.ValidationResult = Rejected("reason", "message") -} diff --git a/internal/modules/generic/normalize/noop/typed_test.go b/internal/modules/generic/normalize/noop/typed_test.go deleted file mode 100644 index 52141b4..0000000 --- a/internal/modules/generic/normalize/noop/typed_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package noop - -import ( - "context" - "testing" - - "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" -) - -func TestTypedNormalizerPreservesValue(t *testing.T) { - normalizer := NewTyped[string]() - result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}}) - if err != nil || result.Value != "value" { - t.Fatalf("result=%#v err=%v", result, err) - } -} diff --git a/internal/modules/integration/dnd_spells_runner_test.go b/internal/modules/integration/dnd_spells_runner_test.go index bafb89b..e9ca52e 100644 --- a/internal/modules/integration/dnd_spells_runner_test.go +++ b/internal/modules/integration/dnd_spells_runner_test.go @@ -164,87 +164,6 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) { } } -func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) { - raw := readDNDSpellsFixture(t) - resolved := resolveDNDSpellsPipeline(t) - resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet( - "Mira: wizard who can cast Lightning Bolt", - "", - ) - llmClient := &fakeSpellsLLMClient{ - response: extractionResponse{SpellCasts: []spellCastResponse{}}, - } - - output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ - RawInput: raw, - }) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs)) - } - response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content) - if len(response.SpellCasts) != 0 { - t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts) - } - if len(llmClient.requests) != 1 { - t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests)) - } - request := llmClient.requests[0] - if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion { - t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion) - } - if got := string(request.Inputs["party"].Content); !strings.Contains(got, "Lightning Bolt") { - t.Fatalf("party input = %q, want party-reference-only spell in reference input", got) - } - if output.Manifest.ValidationStatus != "approved" { - t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus) - } -} - -func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) { - raw := readDNDSpellsFixture(t) - resolved := resolveDNDSpellsPipeline(t) - llmClient := &fakeSpellsLLMClient{ - response: extractionResponse{ - SpellCasts: []spellCastResponse{ - { - Caster: "Aria", - Spell: "Cure Wounds", - Effect: "Heals an injured ally.", - NarrativeDescription: "Aria restores the fighter after the fight.", - SourceRefs: responseSourceRefs("spell-session", 999, 999), - }, - }, - }, - } - - output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ - RawInput: raw, - }) - if err != nil { - t.Fatalf("Run() error = %v, want nil", err) - } - if len(output.NormalizeOutputs) != 1 { - t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs)) - } - response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content) - if len(response.SpellCasts) != 1 { - t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts)) - } - if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" { - t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID) - } - if len(output.Rejected) != 0 { - t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected)) - } - if output.Manifest.ValidationStatus != "approved" { - t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus) - } -} - func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet { slots := make(map[string]contracts.ResolvedReferenceSlot) if strings.TrimSpace(party) != "" { @@ -282,22 +201,6 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet return contracts.ReferenceSet{Slots: slots} } -func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) { - raw := readDNDSpellsFixture(t) - resolved := resolveDNDSpellsPipeline(t) - llmClient := &fakeSpellsLLMClient{response: extractionResponse{}} - - output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{ - RawInput: raw, - }) - if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") { - t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err) - } - if len(output.NormalizeOutputs) != 0 { - t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs)) - } -} - func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig { t.Helper()