601 lines
25 KiB
Go
601 lines
25 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io/fs"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
|
spellshape "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/shape"
|
|
spellsourcerefs "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_refs"
|
|
spellrelatedness "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/spells/source_relatedness"
|
|
validjson "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json"
|
|
validjsonschema "gitea.maximumdirect.net/eric/notarius/internal/modules/generic/validate/valid_json_schema"
|
|
)
|
|
|
|
func TestProductionCompatibilitySnapshot(t *testing.T) {
|
|
normalizedOptions, err := normalizeOptions(Options{})
|
|
if err != nil {
|
|
t.Fatalf("normalizeOptions() error = %v, want nil", err)
|
|
}
|
|
if normalizedOptions.Catalog.Inputs != normalizedOptions.Registries.Inputs ||
|
|
normalizedOptions.Catalog.Chunkers != normalizedOptions.Registries.Chunkers ||
|
|
normalizedOptions.Catalog.Extractors != normalizedOptions.Registries.Extractors ||
|
|
normalizedOptions.Catalog.Mergers != normalizedOptions.Registries.Mergers ||
|
|
normalizedOptions.Catalog.Normalizers != normalizedOptions.Registries.Normalizers ||
|
|
normalizedOptions.Catalog.Validators != normalizedOptions.Registries.Validators ||
|
|
normalizedOptions.Catalog.ValidatorChains != normalizedOptions.Registries.ValidatorChains ||
|
|
normalizedOptions.Catalog.Outputs != normalizedOptions.Registries.Outputs {
|
|
t.Fatal("production catalog and execution registries do not share one composition")
|
|
}
|
|
if normalizedOptions.LLMClientFactory == nil {
|
|
t.Fatal("production LLM client factory is nil")
|
|
}
|
|
|
|
registries, err := productionRegistries()
|
|
if err != nil {
|
|
t.Fatalf("productionRegistries() error = %v, want nil", err)
|
|
}
|
|
|
|
keySnapshots := []struct {
|
|
name string
|
|
got []string
|
|
want []string
|
|
}{
|
|
{name: "inputs", got: registries.Inputs.RegisteredKeys(), want: []string{"seriatim"}},
|
|
{name: "chunkers", got: registries.Chunkers.RegisteredKeys(), want: []string{"dnd/scenes", "generic"}},
|
|
{name: "extractors", got: registries.Extractors.RegisteredKeys(), want: []string{"dnd/spells"}},
|
|
{name: "mergers", got: registries.Mergers.RegisteredKeys(), want: []string{"appendorder"}},
|
|
{name: "normalizers", got: registries.Normalizers.RegisteredKeys(), want: []string{"noop"}},
|
|
{name: "validators", got: registries.Validators.RegisteredKeys(), want: []string{
|
|
"extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
|
"generic/always_accept", "generic/always_reject", "generic/valid_json", "generic/valid_json_schema",
|
|
}},
|
|
{name: "outputs", got: registries.Outputs.RegisteredKeys(), want: []string{"json"}},
|
|
}
|
|
for _, snapshot := range keySnapshots {
|
|
t.Run(snapshot.name, func(t *testing.T) {
|
|
if !reflect.DeepEqual(snapshot.got, snapshot.want) {
|
|
t.Fatalf("registered keys = %#v, want compatibility snapshot %#v", snapshot.got, snapshot.want)
|
|
}
|
|
})
|
|
}
|
|
|
|
wantChain := []pipeline.ModuleBinding{
|
|
pipeline.Binding(validjson.Key),
|
|
pipeline.Binding(validjsonschema.Key),
|
|
pipeline.Binding(spellshape.Key),
|
|
pipeline.Binding(spellsourcerefs.Key),
|
|
pipeline.Binding(spellrelatedness.Key),
|
|
}
|
|
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
|
t.Fatalf("spell validator chain = %#v, want compatibility snapshot %#v", got, wantChain)
|
|
}
|
|
|
|
assets, err := productionPromptAssets()
|
|
if err != nil {
|
|
t.Fatalf("productionPromptAssets() error = %v, want nil", err)
|
|
}
|
|
assertAssetNames(t, assets.PromptFS, []string{
|
|
"dnd.scenes/dnd.scenes.yaml",
|
|
"dnd.scenes/instructions.md",
|
|
"dnd.scenes/sharedassets/common-dnd-references.md",
|
|
"dnd.scenes/sharedassets/common-dnd-system.md",
|
|
"dnd.scenes/sharedassets/common-dnd-transcript.md",
|
|
"dnd.scenes/task.md",
|
|
"dnd.spells/dnd.spells.yaml",
|
|
"dnd.spells/instructions.md",
|
|
"dnd.spells/sharedassets/common-dnd-references.md",
|
|
"dnd.spells/sharedassets/common-dnd-system.md",
|
|
"dnd.spells/sharedassets/common-dnd-transcript.md",
|
|
"dnd.spells/task.md",
|
|
})
|
|
assertAssetNames(t, assets.SchemaFS, []string{
|
|
"dnd_scenes.v1.json",
|
|
"dnd_spells_llm.v1.json",
|
|
})
|
|
|
|
identitySnapshot := map[string]map[string]any{
|
|
"scenes": sceneManifestMetadata(t),
|
|
"spells": spellManifestMetadata(t),
|
|
}
|
|
for name, metadata := range identitySnapshot {
|
|
for _, key := range []string{"prompt_id", "prompt_version", "prompt_sha256", "response_schema_key", "response_schema_id", "response_schema_name", "response_schema_version", "response_schema_sha256"} {
|
|
if value, ok := metadata[key].(string); !ok || value == "" {
|
|
t.Fatalf("%s metadata[%q] = %#v, want non-empty identity", name, key, metadata[key])
|
|
}
|
|
}
|
|
}
|
|
if got := []any{
|
|
identitySnapshot["scenes"]["prompt_id"], identitySnapshot["scenes"]["prompt_version"],
|
|
identitySnapshot["scenes"]["response_schema_key"], identitySnapshot["scenes"]["response_schema_id"], identitySnapshot["scenes"]["response_schema_name"], identitySnapshot["scenes"]["response_schema_version"],
|
|
}; !reflect.DeepEqual(got, []any{"dnd.scenes", "v1", "dnd_scenes", "notarius.dnd.scenes", "notarius_dnd_scenes_v1", "v1"}) {
|
|
t.Fatalf("scene identities = %#v, want compatibility snapshot", got)
|
|
}
|
|
if got := []any{
|
|
identitySnapshot["spells"]["prompt_id"], identitySnapshot["spells"]["prompt_version"],
|
|
identitySnapshot["spells"]["response_schema_key"], identitySnapshot["spells"]["response_schema_id"], identitySnapshot["spells"]["response_schema_name"], identitySnapshot["spells"]["response_schema_version"],
|
|
}; !reflect.DeepEqual(got, []any{"dnd.spells", "v1", "dnd_spells", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}) {
|
|
t.Fatalf("spell identities = %#v, want compatibility snapshot", got)
|
|
}
|
|
|
|
fileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells.config.yml"))
|
|
if err != nil {
|
|
t.Fatalf("LoadFileConfig() error = %v, want nil", err)
|
|
}
|
|
cfg := config.Default()
|
|
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
|
|
t.Fatalf("ApplyFileConfig() error = %v, want nil", err)
|
|
}
|
|
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
|
|
if err != nil {
|
|
t.Fatalf("Resolve() error = %v, want nil", err)
|
|
}
|
|
resolved := effective.ResolvedPipeline
|
|
if resolved.Input.Module != "seriatim" || resolved.Chunk.Module != "generic" || resolved.Output.Module != "json" || len(resolved.ArtifactLanes) != 1 {
|
|
t.Fatalf("resolved example = %#v, want maintained production topology", resolved)
|
|
}
|
|
lane := resolved.ArtifactLanes[0]
|
|
if lane.ID != "spells" || lane.Extract.Module != "dnd/spells" || lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
|
|
t.Fatalf("resolved lane = %#v, want maintained spell lane", lane)
|
|
}
|
|
if got := resolvedValidatorKeys(resolved.ValidatorChains, pipeline.StageExtract, "spells", spells.Key); !reflect.DeepEqual(got, []string{
|
|
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
|
}) {
|
|
t.Fatalf("resolved validator keys = %#v, want compatibility snapshot", got)
|
|
}
|
|
|
|
productionFileConfig, err := config.LoadFileConfig(fixturePath(t, "examples/dnd-spells-production.config.yml"))
|
|
if err != nil {
|
|
t.Fatalf("LoadFileConfig(production example) error = %v, want nil", err)
|
|
}
|
|
productionConfig := config.Default()
|
|
if err := productionConfig.ApplyFileConfig(productionFileConfig); err != nil {
|
|
t.Fatalf("ApplyFileConfig(production example) error = %v, want nil", err)
|
|
}
|
|
productionEffective, err := productionConfig.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalogFromRegistries(registries)})
|
|
if err != nil {
|
|
t.Fatalf("Resolve(production example) error = %v, want nil", err)
|
|
}
|
|
productionResolved := productionEffective.ResolvedPipeline
|
|
if productionConfig.Concurrency.TotalLLM != 1 || productionConfig.Concurrency.StageWorkers["extract"] != 1 || !reflect.DeepEqual(productionResolved.Chunk.Options, map[string]any{"max_units": 50}) {
|
|
t.Fatalf("production example concurrency/options = %#v/%#v, want compatibility snapshot", productionConfig.Concurrency, productionResolved.Chunk.Options)
|
|
}
|
|
bindings := productionResolved.ArtifactLanes[0].ExtractReferences.Bindings
|
|
if len(bindings) != 2 || bindings[0].SlotName != "glossary" || bindings[0].Source != "./dnd-spells-glossary.txt" || bindings[1].SlotName != "party" || bindings[1].Source != "./dnd-spells-roster.txt" {
|
|
t.Fatalf("production example reference bindings = %#v, want maintained glossary and party bindings", bindings)
|
|
}
|
|
}
|
|
|
|
func TestMaintainedSeriatimToDNDCompatibilityBundle(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
client contracts.StructuredLLMClient
|
|
wantStatus string
|
|
wantLaneFile bool
|
|
wantRejectedCount int
|
|
}{
|
|
{name: "approved", client: newFakeRunLLMClient(false), wantStatus: "approved", wantLaneFile: true},
|
|
{name: "validator rejection is nonfatal", client: newFakeRunLLMClient(true), wantStatus: "rejected", wantRejectedCount: 1},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
outputDir := t.TempDir()
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", fixturePath(t, "examples/dnd-spells.config.yml"),
|
|
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
|
|
"--output-dir", outputDir,
|
|
"--diagnostics-dir", t.TempDir(),
|
|
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(test.client, nil)})
|
|
if code != 0 {
|
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
|
}
|
|
|
|
runDir := onlyChildDir(t, outputDir)
|
|
wantFiles := []string{"index.json", "manifest.json", "rejected.json", "warnings.json"}
|
|
if test.wantLaneFile {
|
|
wantFiles = append(wantFiles, "lanes/spells.json")
|
|
}
|
|
sort.Strings(wantFiles)
|
|
if got := relativeFileNames(t, runDir); !reflect.DeepEqual(got, wantFiles) {
|
|
t.Fatalf("durable files = %#v, want compatibility snapshot %#v", got, wantFiles)
|
|
}
|
|
|
|
var manifest artifacts.RunManifest
|
|
readJSONFile(t, filepath.Join(runDir, "manifest.json"), &manifest)
|
|
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" {
|
|
t.Fatalf("manifest module provenance = %#v, want maintained production modules", manifest)
|
|
}
|
|
if len(manifest.ArtifactLanes) != 1 || manifest.ArtifactLanes[0].ID != "spells" {
|
|
t.Fatalf("artifact lanes = %#v, want one spells lane", manifest.ArtifactLanes)
|
|
}
|
|
laneManifest := manifest.ArtifactLanes[0]
|
|
if laneManifest.Extractor != "dnd/spells" || laneManifest.Merger != "appendorder" || laneManifest.Normalizer != "noop" {
|
|
t.Fatalf("manifest lane module provenance = %#v, want maintained production modules", laneManifest)
|
|
}
|
|
if len(manifest.Extractors) != 0 || manifest.Merger != "" || manifest.Normalizer != "" {
|
|
t.Fatalf("legacy top-level lane summaries = %#v/%q/%q, want empty compatibility snapshot", manifest.Extractors, manifest.Merger, manifest.Normalizer)
|
|
}
|
|
if manifest.ValidationStatus != test.wantStatus || len(manifest.RejectedOutputs) != test.wantRejectedCount {
|
|
t.Fatalf("manifest outcome = status %q rejected %#v, want %q/%d", manifest.ValidationStatus, manifest.RejectedOutputs, test.wantStatus, test.wantRejectedCount)
|
|
}
|
|
if !reflect.DeepEqual(manifest.SourceDigests, []string{"sha256:1c98d94ae632fb10a2b56f684cd4fb1019cedb1a629e57dc0977cf4a54135be0"}) {
|
|
t.Fatalf("source digests = %#v, want maintained fixture provenance", manifest.SourceDigests)
|
|
}
|
|
if got := manifestValidatorKeys(manifestValidatorChain(t, manifest, pipeline.StageExtract, "spells", spells.Key)); !reflect.DeepEqual(got, []string{
|
|
"generic/valid_json", "generic/valid_json_schema", "extract/dnd/spells/shape", "extract/dnd/spells/source_refs", "extract/dnd/spells/source_relatedness",
|
|
}) {
|
|
t.Fatalf("manifest validator chain = %#v, want compatibility snapshot", got)
|
|
}
|
|
|
|
var index struct {
|
|
ManifestFile string `json:"manifest_file"`
|
|
OutputFiles []struct {
|
|
LaneID string `json:"lane_id"`
|
|
MediaType string `json:"media_type"`
|
|
File string `json:"file"`
|
|
ModuleKey string `json:"module_key"`
|
|
SchemaID string `json:"schema_id"`
|
|
SchemaName string `json:"schema_name"`
|
|
SchemaVersion string `json:"schema_version"`
|
|
} `json:"output_files"`
|
|
RejectedFile string `json:"rejected_file"`
|
|
WarningsFile string `json:"warnings_file"`
|
|
}
|
|
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
|
|
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" {
|
|
t.Fatalf("output index fixed files = %#v, want compatibility snapshot", index)
|
|
}
|
|
if test.wantLaneFile {
|
|
if len(index.OutputFiles) != 1 {
|
|
t.Fatalf("output index entries = %#v, want one", index.OutputFiles)
|
|
}
|
|
wantOutput := struct {
|
|
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
|
|
}{"spells", "application/json", "lanes/spells.json", "noop", "notarius.dnd.spells", "notarius_dnd_spells_v1", "v1"}
|
|
gotOutput := index.OutputFiles[0]
|
|
got := struct {
|
|
LaneID, MediaType, File, ModuleKey, SchemaID, SchemaName, SchemaVersion string
|
|
}{gotOutput.LaneID, gotOutput.MediaType, gotOutput.File, gotOutput.ModuleKey, gotOutput.SchemaID, gotOutput.SchemaName, gotOutput.SchemaVersion}
|
|
if got != wantOutput {
|
|
t.Fatalf("output index entries = %#v, want compatibility snapshot %#v", index.OutputFiles, wantOutput)
|
|
}
|
|
assertJSONEqual(t, readFile(t, filepath.Join(runDir, "lanes/spells.json")), []byte(`{
|
|
"spell_casts": [{
|
|
"caster": "Aria",
|
|
"spell": "Cure Wounds",
|
|
"effect": "Heals a wounded ally.",
|
|
"narrative_description": "Aria casts Cure Wounds.",
|
|
"source_refs": [{"source_id": "session-alpha", "start_unit_id": 1, "end_unit_id": 1}]
|
|
}]
|
|
}`))
|
|
} else if len(index.OutputFiles) != 0 {
|
|
t.Fatalf("output index entries = %#v, want none for rejected lane", index.OutputFiles)
|
|
}
|
|
|
|
var warnings struct {
|
|
Warnings []contracts.Warning `json:"warnings"`
|
|
}
|
|
readJSONFile(t, filepath.Join(runDir, "warnings.json"), &warnings)
|
|
if len(warnings.Warnings) != 0 {
|
|
t.Fatalf("warnings = %#v, want empty compatibility snapshot", warnings.Warnings)
|
|
}
|
|
var rejected struct {
|
|
Rejected []contracts.RejectedOutput `json:"rejected"`
|
|
}
|
|
readJSONFile(t, filepath.Join(runDir, "rejected.json"), &rejected)
|
|
if len(rejected.Rejected) != test.wantRejectedCount {
|
|
t.Fatalf("rejected outputs = %#v, want %d", rejected.Rejected, test.wantRejectedCount)
|
|
}
|
|
if test.wantRejectedCount == 1 {
|
|
got := rejected.Rejected[0]
|
|
if got.Stage != "extract" || got.LaneID != "spells" || got.ModuleKey != "dnd/spells" || got.ChunkID != "chunk-000001" || got.ChunkIndex != 0 || got.ValidatorName != "extract/dnd/spells/source_refs" || got.ReasonCode != "invalid_source_refs" || got.AttemptCount != 1 {
|
|
t.Fatalf("rejection = %#v, want maintained nonfatal validator outcome", got)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProductionLLMCallersShareScheduledClient(t *testing.T) {
|
|
underlying := newBlockingProductionLLMClient()
|
|
scheduler, err := frameworkllm.NewScheduler(1)
|
|
if err != nil {
|
|
t.Fatalf("NewScheduler() error = %v, want nil", err)
|
|
}
|
|
client := frameworkllm.NewScheduledClient(underlying, scheduler)
|
|
doc := &source.SourceDocument{
|
|
ID: "session-alpha", Kind: "transcript", Format: "application/json", Digest: "sha256:source",
|
|
Units: []source.SourceUnit{
|
|
{ID: 1, Kind: "segment", Text: "Aria casts Cure Wounds.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 1}},
|
|
{ID: 2, Kind: "segment", Text: "The spell takes effect.", Ref: source.SourceRef{SourceID: "session-alpha", StartUnitID: 2, EndUnitID: 2}},
|
|
},
|
|
}
|
|
chunk := source.Chunk{
|
|
ID: "session-alpha:chunk:0", SourceID: doc.ID, Index: 0, Ref: source.SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
|
|
Content: []byte(`{"scene":"Aria casts Cure Wounds."}`), MediaType: "application/json", Units: append([]source.SourceUnit(nil), doc.Units...),
|
|
}
|
|
|
|
var started sync.WaitGroup
|
|
started.Add(2)
|
|
errs := make(chan error, 2)
|
|
go func() {
|
|
started.Done()
|
|
chunker, err := scenes.New(client, scenes.Options{})
|
|
if err == nil {
|
|
_, err = chunker.Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
|
|
}
|
|
errs <- err
|
|
}()
|
|
go func() {
|
|
started.Done()
|
|
extractor, err := spells.New(client, spells.Options{})
|
|
if err == nil {
|
|
_, err = extractor.Extract(context.Background(), contracts.TypedExtractionRequest{Source: doc, Chunk: &chunk})
|
|
}
|
|
errs <- err
|
|
}()
|
|
started.Wait()
|
|
|
|
for i := 0; i < 2; i++ {
|
|
<-underlying.entered
|
|
underlying.release <- struct{}{}
|
|
}
|
|
for i := 0; i < 2; i++ {
|
|
if err := <-errs; err != nil {
|
|
t.Fatalf("production LLM caller error = %v, want nil", err)
|
|
}
|
|
}
|
|
if underlying.maxActive != 1 {
|
|
t.Fatalf("maximum concurrent provider calls = %d, want total_llm limit 1", underlying.maxActive)
|
|
}
|
|
sort.Strings(underlying.stageNames)
|
|
if !reflect.DeepEqual(underlying.stageNames, []string{"dnd/scenes", "dnd/spells"}) {
|
|
t.Fatalf("scheduled stage names = %#v, want both production LLM callers", underlying.stageNames)
|
|
}
|
|
}
|
|
|
|
func TestProductionBundlePreservesLaneAndChunkOrder(t *testing.T) {
|
|
configPath := writeTestConfig(t, `version: 2
|
|
pipelines:
|
|
dnd-session:
|
|
input: seriatim
|
|
chunk:
|
|
module: generic
|
|
options:
|
|
max_units: 1
|
|
artifacts:
|
|
zeta:
|
|
extract: dnd/spells
|
|
alpha:
|
|
extract: dnd/spells
|
|
`)
|
|
outputDir := t.TempDir()
|
|
var stdout bytes.Buffer
|
|
var stderr bytes.Buffer
|
|
code := RunWithOptions([]string{
|
|
"run", "dnd-session",
|
|
"--config", configPath,
|
|
"--input", fixturePath(t, "examples/seriatim-minimal-transcript.json"),
|
|
"--output-dir", outputDir,
|
|
"--diagnostics-dir", t.TempDir(),
|
|
}, &stdout, &stderr, Options{LLMClientFactory: fakeLLMFactory(orderingProductionLLMClient{}, nil)})
|
|
if code != 0 {
|
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
|
}
|
|
|
|
runDir := onlyChildDir(t, outputDir)
|
|
var index struct {
|
|
OutputFiles []struct {
|
|
LaneID string `json:"lane_id"`
|
|
} `json:"output_files"`
|
|
}
|
|
readJSONFile(t, filepath.Join(runDir, "index.json"), &index)
|
|
if len(index.OutputFiles) != 2 {
|
|
t.Fatalf("output index entries = %#v, want two lanes", index.OutputFiles)
|
|
}
|
|
if got := []string{index.OutputFiles[0].LaneID, index.OutputFiles[1].LaneID}; !reflect.DeepEqual(got, []string{"alpha", "zeta"}) {
|
|
t.Fatalf("output lane order = %#v, want resolved lane order", got)
|
|
}
|
|
for _, laneID := range []string{"alpha", "zeta"} {
|
|
var payload struct {
|
|
SpellCasts []struct {
|
|
Spell string `json:"spell"`
|
|
SourceRefs []source.SourceRef `json:"source_refs"`
|
|
} `json:"spell_casts"`
|
|
}
|
|
readJSONFile(t, filepath.Join(runDir, "lanes", laneID+".json"), &payload)
|
|
if len(payload.SpellCasts) != 2 {
|
|
t.Fatalf("lane %q spell casts = %#v, want one per source chunk", laneID, payload.SpellCasts)
|
|
}
|
|
got := []any{
|
|
payload.SpellCasts[0].Spell, payload.SpellCasts[0].SourceRefs[0].StartUnitID,
|
|
payload.SpellCasts[1].Spell, payload.SpellCasts[1].SourceRefs[0].StartUnitID,
|
|
}
|
|
if !reflect.DeepEqual(got, []any{"Cure Wounds", 1, "Shield", 2}) {
|
|
t.Fatalf("lane %q chunk handoff order = %#v, want source chunk order", laneID, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
type blockingProductionLLMClient struct {
|
|
mu sync.Mutex
|
|
active int
|
|
maxActive int
|
|
stageNames []string
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
}
|
|
|
|
type orderingProductionLLMClient struct{}
|
|
|
|
func (orderingProductionLLMClient) CompleteStructured(_ context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
material := req.Inputs["transcript"]
|
|
unitID := 1
|
|
spellName := "Cure Wounds"
|
|
if strings.Contains(string(material.Content), "Shield") {
|
|
unitID = 2
|
|
spellName = "Shield"
|
|
}
|
|
payload, err := json.Marshal(map[string]any{
|
|
"spell_casts": []map[string]any{{
|
|
"caster": "Aria",
|
|
"spell": spellName,
|
|
"effect": "Fixture effect.",
|
|
"narrative_description": "Fixture spell cast.",
|
|
"source_refs": []map[string]any{{
|
|
"start_unit_id": unitID,
|
|
"end_unit_id": unitID,
|
|
}},
|
|
}},
|
|
})
|
|
if err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
if err := json.Unmarshal(payload, out); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: payload}, nil
|
|
}
|
|
|
|
func newBlockingProductionLLMClient() *blockingProductionLLMClient {
|
|
return &blockingProductionLLMClient{entered: make(chan struct{}, 2), release: make(chan struct{}, 2)}
|
|
}
|
|
|
|
func (client *blockingProductionLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
|
client.mu.Lock()
|
|
client.active++
|
|
if client.active > client.maxActive {
|
|
client.maxActive = client.active
|
|
}
|
|
client.stageNames = append(client.stageNames, req.StageName)
|
|
client.mu.Unlock()
|
|
client.entered <- struct{}{}
|
|
|
|
select {
|
|
case <-ctx.Done():
|
|
return contracts.StructuredCompletionResponse{}, ctx.Err()
|
|
case <-client.release:
|
|
}
|
|
|
|
client.mu.Lock()
|
|
client.active--
|
|
client.mu.Unlock()
|
|
|
|
var payload []byte
|
|
switch req.StageName {
|
|
case scenes.Key:
|
|
payload = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Spell","primary_mode":"Narrative","main_participants":["Aria"],"summary":"Aria casts a spell.","boundary_note":"Complete source.","boundary_confidence":"High"}],"boundary_caveats":[]}`)
|
|
case spells.Key:
|
|
payload = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Healing","narrative_description":"Aria casts Cure Wounds.","source_refs":[{"start_unit_id":1,"end_unit_id":1}]}]}`)
|
|
}
|
|
if err := json.Unmarshal(payload, out); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
return contracts.StructuredCompletionResponse{Content: payload}, nil
|
|
}
|
|
|
|
func sceneManifestMetadata(t *testing.T) map[string]any {
|
|
t.Helper()
|
|
chunker, err := scenes.New(orderingProductionLLMClient{}, scenes.Options{})
|
|
if err != nil {
|
|
t.Fatalf("construct scene chunker: %v", err)
|
|
}
|
|
return chunker.ManifestMetadata()
|
|
}
|
|
|
|
func spellManifestMetadata(t *testing.T) map[string]any {
|
|
t.Helper()
|
|
extractor, err := spells.New(orderingProductionLLMClient{}, spells.Options{})
|
|
if err != nil {
|
|
t.Fatalf("construct spell extractor: %v", err)
|
|
}
|
|
return extractor.ManifestMetadata()
|
|
}
|
|
|
|
func assertAssetNames(t *testing.T, getFS func() (fs.FS, error), want []string) {
|
|
t.Helper()
|
|
fSys, err := getFS()
|
|
if err != nil {
|
|
t.Fatalf("asset filesystem error = %v, want nil", err)
|
|
}
|
|
var got []string
|
|
if err := fs.WalkDir(fSys, ".", func(path string, entry fs.DirEntry, err error) error {
|
|
if err == nil && !entry.IsDir() {
|
|
got = append(got, path)
|
|
}
|
|
return err
|
|
}); err != nil {
|
|
t.Fatalf("walk assets: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("asset names = %#v, want compatibility snapshot %#v", got, want)
|
|
}
|
|
}
|
|
|
|
func resolvedValidatorKeys(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID, moduleKey string) []string {
|
|
for _, chain := range chains {
|
|
if chain.Stage == stage && chain.LaneID == laneID && chain.ModuleKey == moduleKey {
|
|
keys := make([]string, 0, len(chain.Validators))
|
|
for _, validator := range chain.Validators {
|
|
keys = append(keys, validator.Binding.Module)
|
|
}
|
|
return keys
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func relativeFileNames(t *testing.T, root string) []string {
|
|
t.Helper()
|
|
var names []string
|
|
if err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error {
|
|
if err != nil || entry.IsDir() {
|
|
return err
|
|
}
|
|
rel, err := filepath.Rel(root, path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
names = append(names, filepath.ToSlash(rel))
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("walk durable output: %v", err)
|
|
}
|
|
sort.Strings(names)
|
|
return names
|
|
}
|
|
|
|
func assertJSONEqual(t *testing.T, got, want []byte) {
|
|
t.Helper()
|
|
var gotValue any
|
|
var wantValue any
|
|
if err := json.Unmarshal(got, &gotValue); err != nil {
|
|
t.Fatalf("unmarshal actual JSON: %v", err)
|
|
}
|
|
if err := json.Unmarshal(want, &wantValue); err != nil {
|
|
t.Fatalf("unmarshal expected JSON: %v", err)
|
|
}
|
|
if !reflect.DeepEqual(gotValue, wantValue) {
|
|
t.Fatalf("JSON = %#v, want compatibility snapshot %#v", gotValue, wantValue)
|
|
}
|
|
}
|