package cli import ( "context" "encoding/json" "fmt" "reflect" "strings" "sync" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/artifacts" "gitea.maximumdirect.net/eric/notarius/internal/core/config" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd" scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions" sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions" scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions" ) func TestProductionSceneDescriptionWorkflow(t *testing.T) { components := productionTestComponents(t) cfg := config.Default() cfg.Pipelines["scene-descriptions"] = pipeline.PipelineProfile{ ID: "scene-descriptions", Input: pipeline.Binding("seriatim"), Chunk: pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}}, Output: pipeline.Binding("json"), Artifacts: map[string]pipeline.ArtifactLaneProfile{ "scene-descriptions": { Extract: pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"}, Normalize: pipeline.Binding(scenenormalize.Key), }, }, } effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "scene-descriptions", Catalog: catalogFromRegistries(components.registries)}) if err != nil { t.Fatalf("Resolve() error = %v", err) } lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0] if lane.ArtifactKind != dnd.SceneDescriptionListKind || lane.Extract.Module != sceneextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != scenenormalize.Key { t.Fatalf("resolved lane = %#v, want production scene-description composition", lane) } if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 { t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences) } llmClient := &sceneDescriptionLLM{} prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient}) if err != nil { t.Fatalf("Prepare() error = %v", err) } output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{ Prepared: prepared, RawInput: readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"), ChunkCacheMode: pipeline.ChunkCacheBypass, }) if err != nil { t.Fatalf("Run() error = %v", err) } if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 { t.Fatalf("run output = %#v, want one approved normalized artifact", output) } wantProfiles := []artifacts.LLMProfileManifest{{ ID: "scene-description-profile", Provider: "promptkit", Model: "deterministic", }} if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) { t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles) } normalizedOutput := output.NormalizeOutputs[0] if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion { t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput) } var value dnd.SceneDescriptionList if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil { t.Fatalf("decode normalized artifact: %v", err) } want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{ {ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."}, {ID: "chunk-000002", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}, Kind: dnd.SceneKindCombat, Title: "Bandit mage casts Shield", Summary: "The bandit mage casts Shield."}, }} if !reflect.DeepEqual(value, want) { t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want) } durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json") if !reflect.DeepEqual(durable, want) { t.Fatalf("durable output payload = %#v, want %#v", durable, want) } if len(output.Warnings) != 0 { t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings) } } type sceneDescriptionLLM struct { mu sync.Mutex profile *artifacts.LLMProfileManifest } func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { if err := ctx.Err(); err != nil { return contracts.StructuredCompletionResponse{}, err } if req.PromptID != sceneextract.PromptID { return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID) } transcript := string(req.Inputs["transcript"].Content) var content string switch { case strings.Contains(transcript, "Cure Wounds"): content = `{"kind":"narrative","title":" Aria casts Cure Wounds ","summary":" Aria casts Cure Wounds. "}` case strings.Contains(transcript, "Shield"): content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}` default: return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript) } if err := json.Unmarshal([]byte(content), out); err != nil { return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err) } profile := artifacts.LLMProfileManifest{ ID: req.ProfileID, Provider: "promptkit", Model: "deterministic", } client.mu.Lock() client.profile = &profile client.mu.Unlock() return contracts.StructuredCompletionResponse{ Content: []byte(content), Provider: profile.Provider, Model: profile.Model, ProfileID: profile.ID, }, nil } func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest { client.mu.Lock() defer client.mu.Unlock() if client.profile == nil { return nil } return []artifacts.LLMProfileManifest{*client.profile} }