Enforce spell catalog validation in the D&D pipeline
This commit is contained in:
@@ -38,6 +38,7 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
||||
assertProductionContains(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
|
||||
assertProductionContains(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
||||
assertProductionContains(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||
"extract/dnd/spells/catalog",
|
||||
"extract/dnd/spells/shape",
|
||||
"extract/dnd/spells/source_refs",
|
||||
"extract/dnd/spells/source_relatedness",
|
||||
@@ -54,6 +55,7 @@ func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
||||
pipeline.Binding("generic/valid_json"),
|
||||
pipeline.Binding("generic/valid_json_schema"),
|
||||
pipeline.Binding("extract/dnd/spells/shape"),
|
||||
pipeline.Binding("extract/dnd/spells/catalog"),
|
||||
pipeline.Binding("extract/dnd/spells/source_refs"),
|
||||
pipeline.Binding("extract/dnd/spells/source_relatedness"),
|
||||
}
|
||||
@@ -124,6 +126,29 @@ func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionSpellValidatorsPrepareFromMaterializedCatalog(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
||||
effective, err := loadMaintainedExample(t, configPath).Resolve(resolveInputForMaintainedExample(components, "dnd-session"))
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production spell configuration: %v", err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: configPath,
|
||||
WorkingDir: filepath.Dir(configPath),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize production spell references: %v", err)
|
||||
}
|
||||
items := materialized.ArtifactLanes[0].ExtractReferences.ReferenceSet.Slots["spell_catalog"].Items
|
||||
if len(items) != 1 || items[0].MediaType != "application/json" || len(items[0].Content) == 0 {
|
||||
t.Fatalf("materialized spell catalog items = %#v, want one JSON item", items)
|
||||
}
|
||||
if _, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||
t.Fatalf("prepare production spell pipeline from materialized catalog: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProductionLLMClientFactoriesBuildOfflineRuntime(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
factories := []struct {
|
||||
|
||||
161
internal/cli/spell_catalog_retry_contract_test.go
Normal file
161
internal/cli/spell_catalog_retry_contract_test.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||
)
|
||||
|
||||
func TestProductionSpellCatalogValidationRetries(t *testing.T) {
|
||||
const retries = 2
|
||||
tests := []struct {
|
||||
name string
|
||||
responses []string
|
||||
wantCalls int
|
||||
wantRejected bool
|
||||
wantSpell string
|
||||
wantWarningCode string
|
||||
}{
|
||||
{
|
||||
name: "unknown spell remains rejected after exhaustion",
|
||||
responses: []string{
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
},
|
||||
wantCalls: retries + 1,
|
||||
wantRejected: true,
|
||||
},
|
||||
{
|
||||
name: "overlay spell becomes valid on retry",
|
||||
responses: []string{
|
||||
productionSpellResponse("Unknown Spell"),
|
||||
productionSpellResponse("Aegis of Emberfall"),
|
||||
},
|
||||
wantCalls: 2,
|
||||
wantSpell: "Aegis of Emberfall",
|
||||
wantWarningCode: "spell_not_near_source",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
components := productionTestComponents(t)
|
||||
configPath := repositoryPath("examples", "dnd-spells-production.config.yml")
|
||||
cfg := loadMaintainedExample(t, configPath)
|
||||
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(components.registries)})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve production configuration: %v", err)
|
||||
}
|
||||
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalogFromRegistries(components.registries), pipeline.ReferenceMaterializationOptions{
|
||||
ConfigPath: configPath,
|
||||
WorkingDir: repositoryPath("examples"),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("materialize production references: %v", err)
|
||||
}
|
||||
materialized.ArtifactLanes[0].Extract.Retries = retries
|
||||
|
||||
llmClient := &catalogRetryLLMClient{responses: tt.responses}
|
||||
prepared, err := pipeline.Prepare(materialized, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare production pipeline: %v", err)
|
||||
}
|
||||
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
||||
Prepared: prepared,
|
||||
RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
|
||||
ChunkCacheMode: pipeline.ChunkCacheBypass,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Run() error = %v, want nil", err)
|
||||
}
|
||||
if calls := llmClient.CallCount(); calls > retries+1 || calls != tt.wantCalls {
|
||||
t.Fatalf("LLM calls = %d, want %d and no more than %d", calls, tt.wantCalls, retries+1)
|
||||
}
|
||||
|
||||
if tt.wantRejected {
|
||||
if len(output.Rejected) != 1 || len(output.NormalizeOutputs) != 0 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want one nonfatal rejection and no merge output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
rejection := output.Rejected[0]
|
||||
if rejection.ReasonCode != "unknown_spell" || rejection.AttemptCount != retries+1 {
|
||||
t.Fatalf("rejection = %#v, want exhausted unknown-spell rejection", rejection)
|
||||
}
|
||||
if len(output.Warnings) != 0 {
|
||||
t.Fatalf("warnings = %#v, want no warnings from rejected attempts", output.Warnings)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
|
||||
t.Fatalf("rejected = %#v normalized = %#v, want only accepted output", output.Rejected, output.NormalizeOutputs)
|
||||
}
|
||||
var value dnd.SpellList
|
||||
if err := json.Unmarshal(output.NormalizeOutputs[0].Artifact.Content, &value); err != nil {
|
||||
t.Fatalf("decode normalized spell list: %v", err)
|
||||
}
|
||||
if len(value.SpellCasts) != 1 || value.SpellCasts[0].Spell != tt.wantSpell {
|
||||
t.Fatalf("normalized spell list = %#v, want accepted overlay spell", value)
|
||||
}
|
||||
if len(output.Warnings) != 1 || output.Warnings[0].ReasonCode != tt.wantWarningCode {
|
||||
t.Fatalf("warnings = %#v, want only accepted-attempt warning", output.Warnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type catalogRetryLLMClient struct {
|
||||
mu sync.Mutex
|
||||
responses []string
|
||||
calls int
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if req.PromptID != spells.PromptID {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||
}
|
||||
client.mu.Lock()
|
||||
index := client.calls
|
||||
client.calls++
|
||||
client.mu.Unlock()
|
||||
if index >= len(client.responses) {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("missing fake response %d", index)
|
||||
}
|
||||
content := []byte(client.responses[index])
|
||||
if err := json.Unmarshal(content, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||
}
|
||||
|
||||
func (client *catalogRetryLLMClient) CallCount() int {
|
||||
client.mu.Lock()
|
||||
defer client.mu.Unlock()
|
||||
return client.calls
|
||||
}
|
||||
|
||||
func productionSpellResponse(name string) string {
|
||||
content, err := json.Marshal(dnd.SpellList{SpellCasts: []dnd.SpellCast{{
|
||||
Caster: "Aria",
|
||||
Spell: name,
|
||||
Effect: "The spell takes effect.",
|
||||
NarrativeDescription: "Aria casts the spell.",
|
||||
SourceRefs: []source.SourceRef{{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
|
||||
}}})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return string(content)
|
||||
}
|
||||
Reference in New Issue
Block a user