343 lines
15 KiB
Go
343 lines
15 KiB
Go
package integration_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
|
combatcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/combatturns"
|
|
npccodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/npcs"
|
|
combatextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/combatturns"
|
|
combatnormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/combatturns"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/identity"
|
|
npcregistry "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/npcs/registry"
|
|
combatshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/combatturns/shape"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/seriatim/input/transcript"
|
|
)
|
|
|
|
func TestProductionCombatPipelineRetriesMergesNormalizesAndWritesJSON(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
catalog := moduleCatalog(registries)
|
|
npcPayload := combatTestNPCPayload(t)
|
|
configValue := combatOnlyConfig()
|
|
effective, err := configValue.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-combat-fixture",
|
|
Catalog: catalog,
|
|
ReferenceOverrides: []pipeline.ReferenceBinding{
|
|
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npcs", Source: npcPayload.path, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npcs", Source: npcPayload.path, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
materialized, warnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err != nil || len(warnings) != 0 {
|
|
t.Fatalf("MaterializeReferences() error = %v warnings = %#v, want no warnings", err, warnings)
|
|
}
|
|
|
|
client := &fakeCombatLLMClient{responses: []string{
|
|
combatTestInvalidEnumResponse("unsupported", "attack"),
|
|
combatTestTurnResponse("mira thorn", "turn", "watches", "mira thorn", 1, 1),
|
|
combatTestTurnResponse("Mira Thorn", "reaction", "asks", "Hooded Guard", 2, 2),
|
|
combatTestTurnResponse("Hooded Guard", "turn", "attacks", "mira thorn", 3, 3),
|
|
}}
|
|
prepared, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
|
}
|
|
fingerprints := prepared.CheckpointFingerprints()
|
|
assertCombatFingerprint(t, fingerprints, "extract:combat:"+combatextract.Key+":npc_registry")
|
|
assertCombatFingerprint(t, fingerprints, "normalize:combat:"+combatnormalize.Key+":npc_registry")
|
|
if len(fingerprints) == 0 {
|
|
t.Fatal("checkpoint fingerprints = empty, want production combat identities")
|
|
}
|
|
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
|
Prepared: prepared,
|
|
RawInput: readNPCFixture(t),
|
|
ExtractWorkers: 1,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want nil", err)
|
|
}
|
|
if len(client.requests) != 4 || len(output.Rejected) != 0 {
|
|
t.Fatalf("LLM requests = %d rejected = %#v, want one retry and no durable rejection", len(client.requests), output.Rejected)
|
|
}
|
|
if output.Manifest.ValidationStatus != "approved" || len(output.NormalizeOutputs) != 1 || len(output.OutputFiles) == 0 {
|
|
t.Fatalf("manifest/output = %#v / %#v, want approved combat JSON output", output.Manifest, output.OutputFiles)
|
|
}
|
|
serialized := output.NormalizeOutputs[0]
|
|
if serialized.LaneID != "combat" || serialized.NormalizerKey != combatnormalize.Key || serialized.Artifact.Schema.ID != combatcodec.SchemaID {
|
|
t.Fatalf("serialized combat output = %#v, want production combat lane schema", serialized)
|
|
}
|
|
value, err := combatcodec.New().Decode(serialized.Artifact.Content)
|
|
if err != nil {
|
|
t.Fatalf("Decode(combat output) error = %v", err)
|
|
}
|
|
if len(value.CombatTurns) != 3 || value.CombatTurns[0].Actor != "Mira Thorn" || value.CombatTurns[0].Actions[0].Targets[0] != "Mira Thorn" || value.CombatTurns[1].Actor != "Mira Thorn" || value.CombatTurns[2].Actor != "Hooded Guard" {
|
|
t.Fatalf("normalized combat output = %#v, want ordered canonical actors and targets", value)
|
|
}
|
|
for _, turn := range value.CombatTurns {
|
|
for _, ref := range turn.SourceRefs {
|
|
if ref.SourceID != "npc-session" {
|
|
t.Fatalf("combat evidence ref = %#v, want current transcript source", ref)
|
|
}
|
|
}
|
|
}
|
|
if !hasCombatWarning(output.Warnings, "combat_turn_not_near_source") || !hasCombatWarning(output.Warnings, combatnormalize.ReasonCodeActorCanonicalized) || !hasCombatWarning(output.Warnings, combatnormalize.ReasonCodeTargetCanonicalized) {
|
|
t.Fatalf("warnings = %#v, want relatedness and registry normalization warnings", output.Warnings)
|
|
}
|
|
if len(output.Manifest.References) != 2 {
|
|
t.Fatalf("manifest references = %#v, want separate extract and normalize provenance", output.Manifest.References)
|
|
}
|
|
lane := output.Manifest.ArtifactLanes[0]
|
|
if lane.ID != "combat" || lane.Extractor != combatextract.Key || lane.Merger != pipeline.DefaultMergeModule || lane.Normalizer != combatnormalize.Key {
|
|
t.Fatalf("manifest lane = %#v, want complete combat composition", lane)
|
|
}
|
|
extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any)
|
|
if !ok || extractorMetadata["npc_count"] != 2 || extractorMetadata["npc_registry_digest"] == "" {
|
|
t.Fatalf("extractor metadata = %#v, want registry digest and count", lane.Metadata)
|
|
}
|
|
normalizerMetadata, ok := lane.Metadata["normalizer"].(map[string]any)
|
|
if !ok || normalizerMetadata["npc_count"] != 2 || normalizerMetadata["normalization_policy"] != combatnormalize.NormalizationPolicy {
|
|
t.Fatalf("normalizer metadata = %#v, want registry and normalization policy", lane.Metadata)
|
|
}
|
|
var combatFile *contracts.OutputFile
|
|
for index := range output.OutputFiles {
|
|
if output.OutputFiles[index].Name == "lanes/combat.json" {
|
|
combatFile = &output.OutputFiles[index]
|
|
break
|
|
}
|
|
}
|
|
if combatFile == nil || combatFile.ContentType != combatcodec.MediaType {
|
|
t.Fatalf("output files = %#v, want lanes/combat.json JSON payload", output.OutputFiles)
|
|
}
|
|
}
|
|
|
|
func TestProductionCombatPipelineAttributesExhaustedInvalidEnumsToShapeValidation(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
configValue := combatOnlyConfig()
|
|
profile := configValue.Pipelines["dnd-combat-fixture"]
|
|
profile.Chunk.Options["max_units"] = 100
|
|
configValue.Pipelines["dnd-combat-fixture"] = profile
|
|
effective, err := configValue.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-combat-fixture",
|
|
Catalog: moduleCatalog(registries),
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
client := &fakeCombatLLMClient{responses: []string{
|
|
combatTestInvalidEnumResponse("unsupported", "attack"),
|
|
combatTestInvalidEnumResponse("turn", "unsupported"),
|
|
combatTestInvalidEnumResponse("unsupported", "unsupported"),
|
|
}}
|
|
prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: client})
|
|
if err != nil {
|
|
t.Fatalf("Prepare() error = %v, want nil", err)
|
|
}
|
|
output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
|
|
Prepared: prepared,
|
|
RawInput: readNPCFixture(t),
|
|
ExtractWorkers: 1,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Run() error = %v, want non-fatal rejected output", err)
|
|
}
|
|
if len(client.requests) != 3 || len(output.Rejected) != 1 {
|
|
t.Fatalf("LLM requests = %d rejected = %#v, want exhausted retry and one rejection", len(client.requests), output.Rejected)
|
|
}
|
|
rejected := output.Rejected[0]
|
|
if rejected.ReasonCode != combatshape.ReasonCode || rejected.ValidatorName != combatshape.Key || rejected.AttemptCount != 3 {
|
|
t.Fatalf("rejected output = %#v, want exhausted combat shape rejection", rejected)
|
|
}
|
|
if output.Manifest.ValidationStatus != "rejected" || len(output.NormalizeOutputs) != 0 {
|
|
t.Fatalf("output = %#v, want non-fatal rejected combat result without normalized artifacts", output)
|
|
}
|
|
}
|
|
|
|
func TestCombatNormalizerRejectsCampaignReferenceBinding(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
catalog := moduleCatalog(registries)
|
|
referencePath := filepath.Join(t.TempDir(), "party.yml")
|
|
if err := os.WriteFile(referencePath, []byte("Aria: cleric\n"), 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
effective, err := combatOnlyConfig().Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-combat-fixture",
|
|
Catalog: catalog,
|
|
ReferenceOverrides: []pipeline.ReferenceBinding{{
|
|
Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "party", Source: referencePath,
|
|
BindingSource: contracts.ReferenceBindingSourceCLI,
|
|
}},
|
|
})
|
|
if err != nil {
|
|
if !strings.Contains(err.Error(), "party") {
|
|
t.Fatalf("Resolve() error = %v, want undeclared party context", err)
|
|
}
|
|
return
|
|
}
|
|
_, _, err = pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if err == nil || !strings.Contains(err.Error(), "party") || !strings.Contains(err.Error(), "not declared") {
|
|
t.Fatalf("MaterializeReferences() error = %v, want undeclared normalize party binding", err)
|
|
}
|
|
}
|
|
|
|
func TestCombatPreparationRejectsMalformedOrOversizedNPCReferencesBeforeExecution(t *testing.T) {
|
|
registries := productionNPCRegistries(t)
|
|
catalog := moduleCatalog(registries)
|
|
for _, test := range []struct {
|
|
name string
|
|
content []byte
|
|
}{
|
|
{name: "malformed", content: []byte(`{"npcs":[`)},
|
|
{name: "oversized", content: append([]byte(`{"npcs":[]}`), make([]byte, npcregistry.MaxBytes+1)...)},
|
|
} {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "npcs.json")
|
|
if err := os.WriteFile(path, test.content, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg := combatOnlyConfig()
|
|
effective, err := cfg.Resolve(config.ResolveInput{
|
|
PipelineID: "dnd-combat",
|
|
Catalog: catalog,
|
|
ReferenceOverrides: []pipeline.ReferenceBinding{
|
|
{Stage: pipeline.StageExtract, LaneID: "combat", SlotName: "npcs", Source: path, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
{Stage: pipeline.StageNormalize, LaneID: "combat", SlotName: "npcs", Source: path, BindingSource: contracts.ReferenceBindingSourceCLI},
|
|
},
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want path binding to resolve before content preparation", err)
|
|
}
|
|
materialized, _, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{})
|
|
if test.name == "oversized" {
|
|
if err == nil || !strings.Contains(err.Error(), "limit") {
|
|
t.Fatalf("MaterializeReferences() error = %v, want bounded oversized-reference failure", err)
|
|
}
|
|
return
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("MaterializeReferences() error = %v, want materialization before typed preparation", err)
|
|
}
|
|
client := &fakeCombatLLMClient{}
|
|
if _, err := pipeline.Prepare(materialized, registries, pipeline.ModuleDependencies{LLM: client}); err == nil {
|
|
t.Fatal("Prepare() error = nil, want NPC registry preparation failure")
|
|
}
|
|
if len(client.requests) != 0 {
|
|
t.Fatalf("LLM requests = %d, want no pipeline execution after preparation failure", len(client.requests))
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type combatNPCPayload struct {
|
|
path string
|
|
}
|
|
|
|
func combatTestNPCPayload(t *testing.T) combatNPCPayload {
|
|
t.Helper()
|
|
value := dnd.NPCList{NPCs: []dnd.NPC{
|
|
{ID: identity.DeriveID("Mira Thorn"), Name: "Mira Thorn", SourceRefs: []source.SourceRef{{SourceID: "prior-npc-session", StartUnitID: 1, EndUnitID: 1}}},
|
|
{ID: identity.DeriveID("Hooded Guard"), Name: "Hooded Guard", SourceRefs: []source.SourceRef{{SourceID: "prior-npc-session", StartUnitID: 3, EndUnitID: 3}}},
|
|
}}
|
|
content, err := npccodec.New().Encode(value)
|
|
if err != nil {
|
|
t.Fatalf("Encode(NPC fixture) error = %v", err)
|
|
}
|
|
path := filepath.Join(t.TempDir(), "npc-run", "lanes", "npcs.json")
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(path, content, 0o600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return combatNPCPayload{path: path}
|
|
}
|
|
|
|
func combatOnlyConfig() config.Config {
|
|
cfg := config.Default()
|
|
cfg.Pipelines["dnd-combat"] = pipeline.PipelineProfile{
|
|
Input: pipeline.Binding(transcript.Key),
|
|
Chunk: pipeline.ModuleBinding{Module: pipeline.DefaultChunkModule, Options: map[string]any{"max_units": 2}},
|
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
|
"combat": {
|
|
Extract: pipeline.ModuleBinding{Module: combatextract.Key, Retries: 2},
|
|
Normalize: pipeline.Binding(combatnormalize.Key),
|
|
},
|
|
},
|
|
}
|
|
cfg.Pipelines["dnd-combat-fixture"] = cfg.Pipelines["dnd-combat"]
|
|
return cfg
|
|
}
|
|
|
|
func repositoryPathForIntegration(parts ...string) string {
|
|
_, file, _, _ := runtime.Caller(0)
|
|
return filepath.Join(append([]string{filepath.Dir(file), "..", "..", ".."}, parts...)...)
|
|
}
|
|
|
|
type fakeCombatLLMClient struct {
|
|
mu sync.Mutex
|
|
responses []string
|
|
requests []contracts.StructuredCompletionRequest
|
|
}
|
|
|
|
func (client *fakeCombatLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
client.mu.Lock()
|
|
client.requests = append(client.requests, req)
|
|
index := len(client.requests) - 1
|
|
client.mu.Unlock()
|
|
if index >= len(client.responses) {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("no combat fake response for call %d", index)
|
|
}
|
|
content := []byte(client.responses[index])
|
|
if err := json.Unmarshal(content, out); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate combat fake target: %w", err)
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "combat-fake"}, nil
|
|
}
|
|
|
|
func combatTestInvalidEnumResponse(turnKind, category string) string {
|
|
return fmt.Sprintf(`{"combat_turns":[{"actor":"Aria","turn_kind":%q,"round":1,"actions":[{"category":%q,"declaration":"watches","targets":["Mira"],"resolution":null}],"summary":"invalid candidate","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`, turnKind, category)
|
|
}
|
|
|
|
func combatTestTurnResponse(actor, turnKind, declaration, target string, round, unit int) string {
|
|
return fmt.Sprintf(`{"combat_turns":[{"actor":%q,"turn_kind":%q,"round":%d,"actions":[{"category":"attack","declaration":%q,"targets":[%q],"resolution":"observed"}],"summary":%q,"source_refs":[{"start_unit_id":%d,"end_unit_id":%d}]}]}`, actor, turnKind, round, declaration, target, declaration, unit, unit)
|
|
}
|
|
|
|
func hasCombatWarning(warnings []contracts.Warning, reason string) bool {
|
|
for _, warning := range warnings {
|
|
if warning.ReasonCode == reason {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func assertCombatFingerprint(t *testing.T, fingerprints []pipeline.CheckpointFingerprint, name string) {
|
|
t.Helper()
|
|
for _, fingerprint := range fingerprints {
|
|
if fingerprint.Name == name && strings.HasPrefix(fingerprint.Value, "sha256:") {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("fingerprints = %#v, want %q with semantic digest", fingerprints, name)
|
|
}
|
|
|
|
var _ contracts.StructuredLLMClient = (*fakeCombatLLMClient)(nil)
|