eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove obsolete and misleading tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-18 23:32:26 +00:00
parent 2cba6d4512
commit bbc83ab042
5 changed files with 0 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
internal/framework/contracts/contracts_test.go
View File

@@ -6,7 +6,6 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
)
@@ -18,155 +17,6 @@ var _ Normalizer[fakeArtifact] = fakeNormalizer{}
var _ StructuredLLMClient = fakeLLMClient{}
var _ OutputEncoder = fakeOutputEncoder{}
func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
extractor := fakeExtractor{
key: "generic-extractor",
}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "Source text."},
},
}
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{Source: doc})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if extractor.Key() != "generic-extractor" {
t.Fatalf("Key() = %q, want generic-extractor", extractor.Key())
}
if result.Value.Value != "example" {
t.Fatalf("Value = %q, want example", result.Value.Value)
}
}
func TestFakeChunkerReturnsSourcePlan(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "Source text."},
},
}
chunker := fakeChunker{key: "generic-chunker"}
result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc})
if err != nil {
t.Fatalf("Plan() error = %v, want nil", err)
}
if chunker.Key() != "generic-chunker" {
t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
}
if len(result.Plan.Ranges) != 1 {
t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges))
}
if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan)
}
}
func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "Source text."},
},
}
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
t.Fatalf("Plan() error = %v, want nil", err)
}
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
}
}
func TestFakeExtractorReceivesChunkAndAmbientContext(t *testing.T) {
extractor := fakeExtractor{key: "generic-extractor"}
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
Format: "text/plain",
Digest: "sha256:abc123",
Units: []source.SourceUnit{
{ID: 1, Kind: "section", Text: "First source text."},
{ID: 2, Kind: "section", Text: "Second source text."},
},
}
chunk := source.Chunk{
ID: "source-1:chunk:1",
SourceID: doc.ID,
Index: 1,
Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 2, EndUnitID: 2},
Content: []byte(`{"units":[{"id":2,"kind":"section","text":"Second source text."}]}`),
MediaType: "application/json",
Units: []source.SourceUnit{doc.Units[1]},
}
result, err := extractor.Extract(context.Background(), TypedExtractionRequest{
Source: doc,
Chunk: &chunk,
AmbientContext: map[string]any{"mode": "chunked"},
})
if err != nil {
t.Fatalf("Extract() error = %v, want nil", err)
}
if result.Value.Value != "chunked" {
t.Fatalf("Value = %q, want chunked", result.Value.Value)
}
}
func TestReferenceSetDataTypes(t *testing.T) {
references := ReferenceSet{
Slots: map[string]ResolvedReferenceSlot{
"roster": {
Slot: ReferenceSlot{
Name: "roster",
Description: "Known characters",
Required: true,
AcceptedMediaTypes: []string{"text/plain"},
Multiple: true,
MaxBytes: 4096,
},
Items: []ReferenceItem{
{
SlotName: "roster",
MediaType: "text/plain",
Content: []byte("Aria\nBryn\n"),
Digest: "sha256:reference",
Origin: ReferenceOrigin{
Type: "file",
URI: "file:///tmp/roster.txt",
},
SizeBytes: 10,
BindingSource: ReferenceBindingSourceConfig,
},
},
},
},
}
item := references.Slots["roster"].Items[0]
if item.SlotName != "roster" || item.MediaType != "text/plain" || string(item.Content) != "Aria\nBryn\n" {
t.Fatalf("reference item = %#v, want constructed item fields", item)
}
if item.BindingSource != ReferenceBindingSourceConfig {
t.Fatalf("BindingSource = %q, want %q", item.BindingSource, ReferenceBindingSourceConfig)
}
}
func TestCloneReferenceSlotsEmptyInputReturnsNil(t *testing.T) {
if got := CloneReferenceSlots(nil); got != nil {
t.Fatalf("CloneReferenceSlots(nil) = %#v, want nil", got)
@@ -319,61 +169,6 @@ func TestArtifactSchemaJSONOmitsSchemaContent(t *testing.T) {
}
}
func TestFakeMergeNormalizeAndOutputContracts(t *testing.T) {
extractOutput := ExtractArtifact[fakeArtifact]{LaneID: "generic-lane", ExtractorKey: "generic-extractor", SourceID: "source-1", ChunkID: "source-1:chunk:0", ChunkIndex: 0, Value: fakeArtifact{Value: "example"}}
merger := fakeMerger{key: "generic-merger"}
normalizer := fakeNormalizer{key: "generic-normalizer"}
encoder := fakeOutputEncoder{key: "generic-output"}
merged, err := merger.Merge(context.Background(), TypedMergeRequest[fakeArtifact]{
LaneID: "generic-lane",
ExtractOutputs: []ExtractArtifact[fakeArtifact]{extractOutput},
})
if err != nil {
t.Fatalf("Merge() error = %v, want nil", err)
}
if merger.Key() != "generic-merger" {
t.Fatalf("Merger.Key() = %q, want generic-merger", merger.Key())
}
if merged.Value.Value != "example" {
t.Fatalf("merged value = %q, want example", merged.Value.Value)
}
normalized, err := normalizer.Normalize(context.Background(), TypedNormalizeRequest[fakeArtifact]{
LaneID: "generic-lane",
MergeOutput: MergeArtifact[fakeArtifact]{LaneID: "generic-lane", MergerKey: merger.Key(), SourceID: "source-1", Value: merged.Value},
})
if err != nil {
t.Fatalf("Normalize() error = %v, want nil", err)
}
if normalizer.Key() != "generic-normalizer" {
t.Fatalf("Normalizer.Key() = %q, want generic-normalizer", normalizer.Key())
}
if normalized.Value.Value != "example" {
t.Fatalf("normalized value = %q, want example", normalized.Value.Value)
}
encoded, err := encoder.Encode(context.Background(), OutputRequest{
Manifest: artifacts.RunManifest{RunID: "run-1"},
NormalizeOutputs: []SerializedOutput{{LaneID: "generic-lane", NormalizerKey: normalizer.Key(), SourceID: "source-1", Artifact: SerializedArtifact{Kind: "test/artifact", Schema: ArtifactSchema{ID: "schema-id", Name: "schema-name", Version: "v1"}, MediaType: "application/json", Content: []byte(`{"value":"example"}`)}}},
})
if err != nil {
t.Fatalf("Encode() error = %v, want nil", err)
}
if encoder.Key() != "generic-output" {
t.Fatalf("OutputEncoder.Key() = %q, want generic-output", encoder.Key())
}
if len(encoded.Files) != 1 {
t.Fatalf("len(Files) = %d, want 1", len(encoded.Files))
}
if encoded.Files[0].ContentType != "application/json" {
t.Fatalf("ContentType = %q, want application/json", encoded.Files[0].ContentType)
}
if string(encoded.Files[0].Bytes) != `{"run_id":"run-1","output_count":1}` {
t.Fatalf("Bytes = %s, want encoded output", encoded.Files[0].Bytes)
}
}
func TestOutputFileJSONShapeOmitsBytes(t *testing.T) {
file := OutputFile{
Name: "artifacts/events.json",
@@ -441,24 +236,6 @@ func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPla
}, nil
}
type recordingChunker struct {
key string
request ChunkRequest
}
func (chunker *recordingChunker) Key() string {
return chunker.key
}
func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Plan(ctx, req)
}
type fakeExtractor struct {
key string
}

175
internal/framework/llm/scriptorium_api_test.go
View File

@@ -1,175 +0,0 @@
package llm
import (
"context"
"errors"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium"
)
func TestScriptoriumPublicAPIGrounding(t *testing.T) {
// Keep this compile-time grounding close to the future Notarius adapter so
// dependency upgrades reveal API drift before the runtime cutover.
engine, err := scriptorium.NewEngine(
scriptorium.Config{
PromptDir: "unused-when-prompt-option-is-set",
ProfileDir: "",
SchemaDir: "",
Timeout: time.Second,
},
scriptorium.WithPromptFS(fstest.MapFS{}, "."),
scriptorium.WithProfileFS(fstest.MapFS{}, "."),
scriptorium.WithSchemaFS(fstest.MapFS{}, "."),
scriptorium.WithProfiles(scriptorium.OpenAICompatibleProfile(scriptorium.OpenAICompatibleProfileConfig{
ID: "test-profile",
Endpoint: "http://127.0.0.1:1/v1",
Model: "test-model",
APIKeyRequired: true,
ExtraParams: map[string]any{"mode": "test"},
})),
scriptorium.WithLLMClient(scriptoriumGroundingLLMClient{}),
)
if err != nil {
t.Fatalf("NewEngine() error = %v, want nil", err)
}
if engine == nil {
t.Fatalf("NewEngine() = nil, want engine")
}
var (
_ func(string) scriptorium.Option = scriptorium.WithPromptFile
_ func(string) scriptorium.Option = scriptorium.WithProfileFile
_ func(string) scriptorium.Option = scriptorium.WithSchemaFile
)
req := scriptorium.RunRequest{
PromptID: "dnd.spells",
PromptVersion: "v1",
ProfileID: "test-profile",
APIKey: "request-scoped-secret",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.InlineWithURI("file:///tmp/transcript.json", `{"segments":[]}`),
"glossary": scriptorium.Inline(""),
"roster": scriptorium.File("/tmp/roster.txt"),
},
Vars: map[string]string{
"session_id": "session-1",
},
Execution: &scriptorium.ExecutionTargetOverride{
Model: "override-model",
Temperature: ptr(0.2),
MaxTokens: ptr(100),
TopP: ptr(0.9),
TimeoutSeconds: ptr(30),
ServiceTier: "standard",
ReasoningEffort: "low",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{"provider_option": "value"},
},
Validation: &scriptorium.OutputContract{
Format: scriptorium.FormatJSON,
ValidationMode: scriptorium.ValidationJSONSchema,
SchemaPath: "schemas/dnd_spells.v1.json",
RepairAttempts: 1,
},
Metadata: map[string]string{
"artifact_kind": "dnd_spell",
},
}
if req.Inputs["transcript"].Type != scriptorium.ArtifactRefInline {
t.Fatalf("inline input type = %q, want %q", req.Inputs["transcript"].Type, scriptorium.ArtifactRefInline)
}
if req.Inputs["roster"].Type != scriptorium.ArtifactRefFile {
t.Fatalf("file input type = %q, want %q", req.Inputs["roster"].Type, scriptorium.ArtifactRefFile)
}
result := scriptorium.RunResult{
RunID: "run-1",
Artifact: scriptorium.Artifact{
Name: "output",
ContentType: "application/json",
Body: []byte(`{"ok":true}`),
URI: "inline://output",
Size: int64(len(`{"ok":true}`)),
Hash: "sha256:abc",
},
RawOutput: `{"ok":true}`,
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-prompt-hash",
SelectedProfileID: req.ProfileID,
ModelName: "test-model",
Endpoint: "http://127.0.0.1:1/v1",
EffectiveModelParams: scriptorium.ExecutionTarget{
Model: "test-model",
APIKeyEnv: "SCRIPTORIUM_API_KEY",
ExtraParams: map[string]any{"provider_option": "value"},
ReasoningEffort: "low",
},
InputHashes: map[string]string{
"transcript": "sha256:def",
},
Validation: scriptorium.ValidationResult{
Status: scriptorium.ValidationPassed,
Mode: scriptorium.ValidationJSONSchema,
SchemaPath: req.Validation.SchemaPath,
RepairAttempts: 1,
IsValid: true,
},
Usage: scriptorium.TokenUsage{
PromptTokens: 10,
CompletionTokens: 5,
TotalTokens: 15,
CachedTokens: 3,
CacheWriteTokens: 2,
},
StartTime: time.Unix(1, 0),
EndTime: time.Unix(2, 0),
Duration: time.Second,
}
if result.Validation.Status != scriptorium.ValidationPassed {
t.Fatalf("validation status = %q, want %q", result.Validation.Status, scriptorium.ValidationPassed)
}
if result.Usage.TotalTokens != 15 {
t.Fatalf("total tokens = %d, want 15", result.Usage.TotalTokens)
}
publicErrors := []error{
scriptorium.ErrInvalidConfig,
scriptorium.ErrInvalidRequest,
scriptorium.ErrPromptNotFound,
scriptorium.ErrProfileNotFound,
scriptorium.ErrPromptLoad,
scriptorium.ErrProfileLoad,
scriptorium.ErrArtifactLoad,
scriptorium.ErrPromptRender,
scriptorium.ErrLLMGenerate,
scriptorium.ErrValidation,
}
for _, publicErr := range publicErrors {
if !errors.Is(publicErr, publicErr) {
t.Fatalf("sentinel error does not match itself: %v", publicErr)
}
}
}
type scriptoriumGroundingLLMClient struct{}
func (scriptoriumGroundingLLMClient) Generate(context.Context, scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
return &scriptorium.GenerateResponse{
Content: `{"ok":true}`,
Usage: scriptorium.TokenUsage{
PromptTokens: 1,
CompletionTokens: 1,
TotalTokens: 2,
},
}, nil
}
func ptr[T any](v T) *T {
return &v
}

7
internal/framework/validate/validate_test.go
View File

@@ -2,8 +2,6 @@ package validate
import (
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestApproved(t *testing.T) {
@@ -33,8 +31,3 @@ func TestRejectedTrimsReasonAndMessage(t *testing.T) {
t.Fatalf("Message = %q, want message", result.Message)
}
}
func TestHelpersReturnValidationResults(t *testing.T) {
var _ contracts.ValidationResult = Approved()
var _ contracts.ValidationResult = Rejected("reason", "message")
}

16
internal/modules/generic/normalize/noop/typed_test.go
View File

@@ -1,16 +0,0 @@
package noop
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)
func TestTypedNormalizerPreservesValue(t *testing.T) {
normalizer := NewTyped[string]()
result, err := normalizer.Normalize(context.Background(), contracts.TypedNormalizeRequest[string]{MergeOutput: contracts.MergeArtifact[string]{Value: "value"}})
if err != nil || result.Value != "value" {
t.Fatalf("result=%#v err=%v", result, err)
}
}

97
internal/modules/integration/dnd_spells_runner_test.go
View File

@@ -164,87 +164,6 @@ func TestRunnerPassesPartyAndGlossaryReferencesToDNDSpellsPrompt(t *testing.T) {
}
}
func TestRunnerDoesNotExtractSpellMentionedOnlyInPartyReference(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
resolved.ResolvedPipeline.ArtifactLanes[0].ExtractReferences.ReferenceSet = dndSpellsReferenceSet(
"Mira: wizard who can cast Lightning Bolt",
"",
)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{SpellCasts: []spellCastResponse{}},
}
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want empty spell response output", len(output.NormalizeOutputs))
}
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content)
if len(response.SpellCasts) != 0 {
t.Fatalf("spell_casts = %#v, want no party-reference-only spell casts", response.SpellCasts)
}
if len(llmClient.requests) != 1 {
t.Fatalf("LLM calls = %d, want 1", len(llmClient.requests))
}
request := llmClient.requests[0]
if request.PromptID != spells.PromptID || request.PromptVersion != spells.SchemaVersion {
t.Fatalf("prompt = %q/%q, want %q/%q", request.PromptID, request.PromptVersion, spells.PromptID, spells.SchemaVersion)
}
if got := string(request.Inputs["party"].Content); !strings.Contains(got, "Lightning Bolt") {
t.Fatalf("party input = %q, want party-reference-only spell in reference input", got)
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved empty extraction", output.Manifest.ValidationStatus)
}
}
func TestRunnerCarriesDNDSpellCastWithInvalidSourceRefToSerializedOutput(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{
response: extractionResponse{
SpellCasts: []spellCastResponse{
{
Caster: "Aria",
Spell: "Cure Wounds",
Effect: "Heals an injured ally.",
NarrativeDescription: "Aria restores the fighter after the fight.",
SourceRefs: responseSourceRefs("spell-session", 999, 999),
},
},
},
}
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
})
if err != nil {
t.Fatalf("Run() error = %v, want nil", err)
}
if len(output.NormalizeOutputs) != 1 {
t.Fatalf("len(NormalizeOutputs) = %d, want 1", len(output.NormalizeOutputs))
}
response := decodeRunnerSpellResponse(t, output.NormalizeOutputs[0].Artifact.Content)
if len(response.SpellCasts) != 1 {
t.Fatalf("len(spell_casts) = %d, want 1", len(response.SpellCasts))
}
if response.SpellCasts[0].SourceRefs[0].SourceID != "spell-session" {
t.Fatalf("SourceID = %q, want invalid source ref preserved", response.SpellCasts[0].SourceRefs[0].SourceID)
}
if len(output.Rejected) != 0 {
t.Fatalf("len(Rejected) = %d, want 0", len(output.Rejected))
}
if output.Manifest.ValidationStatus != "approved" {
t.Fatalf("ValidationStatus = %q, want approved", output.Manifest.ValidationStatus)
}
}
func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet {
slots := make(map[string]contracts.ResolvedReferenceSlot)
if strings.TrimSpace(party) != "" {
@@ -282,22 +201,6 @@ func dndSpellsReferenceSet(party string, glossary string) contracts.ReferenceSet
return contracts.ReferenceSet{Slots: slots}
}
func TestRunnerRejectsMalformedDNDSpellsArtifactAtSerializationBoundary(t *testing.T) {
raw := readDNDSpellsFixture(t)
resolved := resolveDNDSpellsPipeline(t)
llmClient := &fakeSpellsLLMClient{response: extractionResponse{}}
output, err := runPreparedPipeline(t, dndSpellsRunnerRegistries(t), resolved.ResolvedPipeline, llmClient, pipeline.RunInput{
RawInput: raw,
})
if err == nil || !strings.Contains(err.Error(), "spell_casts must be present") {
t.Fatalf("Run() error = %v, want invalid spell-list serialization error", err)
}
if len(output.NormalizeOutputs) != 0 {
t.Fatalf("len(NormalizeOutputs) = %d, want no serialized malformed artifact", len(output.NormalizeOutputs))
}
}
func resolveDNDSpellsPipeline(t *testing.T) config.EffectiveConfig {
t.Helper()