notarius/internal/cli/dnd_scene_descriptions_contract_test.go

package cli

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"testing"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
	scenecodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/scenedescriptions"
	sceneextract "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/scenedescriptions"
	scenenormalize "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/normalize/scenedescriptions"
)

func TestProductionSceneDescriptionWorkflow(t *testing.T) {
	components := productionTestComponents(t)
	cfg := config.Default()
	cfg.Pipelines["scene-descriptions"] = pipeline.PipelineProfile{
		ID:     "scene-descriptions",
		Input:  pipeline.Binding("seriatim"),
		Chunk:  pipeline.ModuleBinding{Module: "generic", Options: map[string]any{"max_units": 1}},
		Output: pipeline.Binding("json"),
		Artifacts: map[string]pipeline.ArtifactLaneProfile{
			"scene-descriptions": {
				Extract:   pipeline.ModuleBinding{Module: sceneextract.Key, LLMProfile: "scene-description-profile"},
				Normalize: pipeline.Binding(scenenormalize.Key),
			},
		},
	}
	effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "scene-descriptions", Catalog: catalogFromRegistries(components.registries)})
	if err != nil {
		t.Fatalf("Resolve() error = %v", err)
	}
	lane := effective.ResolvedPipeline.Steps[0].ArtifactLanes[0]
	if lane.ArtifactKind != dnd.SceneDescriptionListKind || lane.Extract.Module != sceneextract.Key || lane.Merge.Module != pipeline.DefaultMergeModule || lane.Normalize.Module != scenenormalize.Key {
		t.Fatalf("resolved lane = %#v, want production scene-description composition", lane)
	}
	if len(lane.ExtractReferences.Bindings) != 0 || len(lane.NormalizeReferences.Bindings) != 0 {
		t.Fatalf("resolved references = %#v / %#v, want no generated or required references", lane.ExtractReferences, lane.NormalizeReferences)
	}

	llmClient := &sceneDescriptionLLM{}
	prepared, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: llmClient})
	if err != nil {
		t.Fatalf("Prepare() error = %v", err)
	}
	output, err := pipeline.New().Run(context.Background(), pipeline.RunInput{
		Prepared:       prepared,
		RawInput:       readRepositoryFile(t, "examples", "seriatim-minimal-transcript.json"),
		ChunkCacheMode: pipeline.ChunkCacheBypass,
	})
	if err != nil {
		t.Fatalf("Run() error = %v", err)
	}
	if output.Manifest.ValidationStatus != "approved" || len(output.Rejected) != 0 || len(output.NormalizeOutputs) != 1 {
		t.Fatalf("run output = %#v, want one approved normalized artifact", output)
	}
	wantProfiles := []artifacts.LLMProfileManifest{{
		ID:       "scene-description-profile",
		Provider: "promptkit",
		Model:    "deterministic",
	}}
	if !reflect.DeepEqual(output.Manifest.LLMProfiles, wantProfiles) {
		t.Fatalf("manifest LLM profiles = %#v, want %#v", output.Manifest.LLMProfiles, wantProfiles)
	}
	normalizedOutput := output.NormalizeOutputs[0]
	if normalizedOutput.NormalizerKey != scenenormalize.Key || normalizedOutput.Artifact.Kind != dnd.SceneDescriptionListKind || normalizedOutput.Artifact.Schema.ID != scenecodec.SchemaID || normalizedOutput.Artifact.Schema.Name != scenecodec.SchemaName || normalizedOutput.Artifact.Schema.Version != scenecodec.SchemaVersion {
		t.Fatalf("normalized output = %#v, want registered durable scene-description schema", normalizedOutput)
	}

	var value dnd.SceneDescriptionList
	if err := json.Unmarshal(normalizedOutput.Artifact.Content, &value); err != nil {
		t.Fatalf("decode normalized artifact: %v", err)
	}
	want := dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{
		{ID: "chunk-000001", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}, Kind: dnd.SceneKindNarrative, Title: "Aria casts Cure Wounds", Summary: "Aria casts Cure Wounds."},
		{ID: "chunk-000002", SourceRef: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}, Kind: dnd.SceneKindCombat, Title: "Bandit mage casts Shield", Summary: "The bandit mage casts Shield."},
	}}
	if !reflect.DeepEqual(value, want) {
		t.Fatalf("normalized scene descriptions = %#v, want %#v", value, want)
	}
	durable := decodeAssembledOutput[dnd.SceneDescriptionList](t, output.OutputFiles, "lanes/scene-descriptions.json")
	if !reflect.DeepEqual(durable, want) {
		t.Fatalf("durable output payload = %#v, want %#v", durable, want)
	}
	if len(output.Warnings) != 0 {
		t.Fatalf("warnings = %#v, want grounded descriptions without warnings", output.Warnings)
	}
}

type sceneDescriptionLLM struct {
	mu      sync.Mutex
	profile *artifacts.LLMProfileManifest
}

func (client *sceneDescriptionLLM) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
	if err := ctx.Err(); err != nil {
		return contracts.StructuredCompletionResponse{}, err
	}
	if req.PromptID != sceneextract.PromptID {
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
	}
	transcript := string(req.Inputs["transcript"].Content)
	var content string
	switch {
	case strings.Contains(transcript, "Cure Wounds"):
		content = `{"kind":"narrative","title":"  Aria casts Cure Wounds  ","summary":"  Aria casts Cure Wounds.  "}`
	case strings.Contains(transcript, "Shield"):
		content = `{"kind":"combat","title":"Bandit mage casts Shield","summary":"The bandit mage casts Shield."}`
	default:
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected transcript material %q", transcript)
	}
	if err := json.Unmarshal([]byte(content), out); err != nil {
		return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate structured response: %w", err)
	}
	profile := artifacts.LLMProfileManifest{
		ID:       req.ProfileID,
		Provider: "promptkit",
		Model:    "deterministic",
	}
	client.mu.Lock()
	client.profile = &profile
	client.mu.Unlock()
	return contracts.StructuredCompletionResponse{
		Content:   []byte(content),
		Provider:  profile.Provider,
		Model:     profile.Model,
		ProfileID: profile.ID,
	}, nil
}

func (client *sceneDescriptionLLM) LLMProfileManifests() []artifacts.LLMProfileManifest {
	client.mu.Lock()
	defer client.mu.Unlock()
	if client.profile == nil {
		return nil
	}
	return []artifacts.LLMProfileManifest{*client.profile}
}