Add production and example composition contract tests
This commit is contained in:
149
internal/cli/example_contract_test.go
Normal file
149
internal/cli/example_contract_test.go
Normal file
@@ -0,0 +1,149 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMaintainedExamplesLoadResolveAndList(t *testing.T) {
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
|
t.Run(example.name, func(t *testing.T) {
|
||||||
|
cfg := loadMaintainedExample(t, example.path)
|
||||||
|
if _, err := cfg.Resolve(resolveInputForMaintainedExample(components, "dnd-session")); err != nil {
|
||||||
|
t.Fatalf("resolve maintained example: %v", err)
|
||||||
|
}
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{"pipelines", "list", "--config", example.path}, &stdout, &stderr, productionOptionsFromComponents(components))
|
||||||
|
if code != 0 || stdout.String() != "dnd-session\n" || stderr.Len() != 0 {
|
||||||
|
t.Fatalf("pipelines list: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaintainedMinimalInvocationProducesJSONBundle(t *testing.T) {
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
|
fake := &productionFakeLLMClient{}
|
||||||
|
options := productionRunOptions(t, fake)
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "dnd-session",
|
||||||
|
"--config", repositoryPath("examples", "dnd-spells.config.yml"),
|
||||||
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||||
|
"--only", "spells", "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||||
|
}, &stdout, &stderr, options)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdout.String(), `pipeline "dnd-session"`) || !strings.Contains(stdout.String(), "outputs=1 rejected=0") {
|
||||||
|
t.Fatalf("stdout=%q, want completed pipeline and counts", stdout.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
runRoot := filepath.Join(outputRoot, productionRunID)
|
||||||
|
index := readProductionJSON[exampleOutputIndex](t, filepath.Join(runRoot, "index.json"))
|
||||||
|
if index.ManifestFile != "manifest.json" || index.RejectedFile != "rejected.json" || index.WarningsFile != "warnings.json" || len(index.OutputFiles) != 1 {
|
||||||
|
t.Fatalf("index = %#v, want one spells output and fixed companion files", index)
|
||||||
|
}
|
||||||
|
entry := index.OutputFiles[0]
|
||||||
|
if entry.LaneID != "spells" || entry.File != "lanes/spells.json" || entry.MediaType != "application/json" || entry.SchemaID != "notarius.dnd.spells" || entry.SchemaVersion != "v1" {
|
||||||
|
t.Fatalf("index output entry = %#v, want spells JSON contract", entry)
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(runRoot, "manifest.json"))
|
||||||
|
if manifest.PipelineID != "dnd-session" || manifest.InputModule != "seriatim" || manifest.Chunker != "generic" || manifest.OutputEncoder != "json" || manifest.ValidationStatus != "approved" || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" {
|
||||||
|
t.Fatalf("manifest = %#v, want approved minimal run", manifest)
|
||||||
|
}
|
||||||
|
if len(manifest.ArtifactLanes) != 1 {
|
||||||
|
t.Fatalf("manifest lanes = %#v, want exactly spells", manifest.ArtifactLanes)
|
||||||
|
}
|
||||||
|
lane := manifest.ArtifactLanes[0]
|
||||||
|
if lane.ID != "spells" || lane.Extractor != "dnd/spells" || lane.Merger != "appendorder" || lane.Normalizer != "noop" {
|
||||||
|
t.Fatalf("manifest lane = %#v, want production spells composition", lane)
|
||||||
|
}
|
||||||
|
|
||||||
|
artifact := readProductionJSON[dnd.SpellList](t, filepath.Join(runRoot, entry.File))
|
||||||
|
if len(artifact.SpellCasts) != 1 || artifact.SpellCasts[0].Spell != "Cure Wounds" || artifact.SpellCasts[0].SourceRefs[0].SourceID != "session-alpha" {
|
||||||
|
t.Fatalf("artifact = %#v, want one source-linked Cure Wounds cast", artifact)
|
||||||
|
}
|
||||||
|
rejected := readProductionJSON[struct {
|
||||||
|
Rejected []json.RawMessage `json:"rejected"`
|
||||||
|
}](t, filepath.Join(runRoot, "rejected.json"))
|
||||||
|
if len(rejected.Rejected) != 0 {
|
||||||
|
t.Fatalf("rejected = %#v, want empty rejection list", rejected.Rejected)
|
||||||
|
}
|
||||||
|
warnings := readProductionJSON[struct {
|
||||||
|
Warnings []json.RawMessage `json:"warnings"`
|
||||||
|
}](t, filepath.Join(runRoot, "warnings.json"))
|
||||||
|
if len(warnings.Warnings) != 0 {
|
||||||
|
t.Fatalf("warnings = %#v, want empty warning list", warnings.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaintainedMalformedInputOnlyRecordsDebugFailureWhenRequested(t *testing.T) {
|
||||||
|
malformed := filepath.Join(t.TempDir(), "malformed.json")
|
||||||
|
if err := os.WriteFile(malformed, []byte("{not valid json"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, debug := range []bool{false, true} {
|
||||||
|
name := "without debug"
|
||||||
|
if debug {
|
||||||
|
name = "with debug"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
|
debugRoot := filepath.Join(t.TempDir(), "debug")
|
||||||
|
options := productionRunOptions(t, &productionFakeLLMClient{})
|
||||||
|
args := []string{
|
||||||
|
"run", "dnd-session",
|
||||||
|
"--config", repositoryPath("examples", "dnd-spells.config.yml"),
|
||||||
|
"--input", malformed, "--chunk_cache", "bypass", "--output-dir", outputRoot,
|
||||||
|
}
|
||||||
|
if debug {
|
||||||
|
args = append(args, "--debug", "--debug-dir", debugRoot)
|
||||||
|
}
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions(args, &stdout, &stderr, options)
|
||||||
|
if code != 1 || stdout.Len() != 0 || !strings.Contains(stderr.String(), "parse input") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
assertAbsent(t, outputRoot)
|
||||||
|
if !debug {
|
||||||
|
assertAbsent(t, debugRoot)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bundle := onlyChildDir(t, debugRoot)
|
||||||
|
report := readProductionJSON[debugbundle.RunReport](t, filepath.Join(bundle, "summary", "run-report.json"))
|
||||||
|
if report.Succeeded || report.PipelineID != "dnd-session" {
|
||||||
|
t.Fatalf("failure report = %#v, want failed dnd-session report", report)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type exampleOutputIndex struct {
|
||||||
|
ManifestFile string `json:"manifest_file"`
|
||||||
|
OutputFiles []exampleOutputIndexEntry `json:"output_files"`
|
||||||
|
RejectedFile string `json:"rejected_file"`
|
||||||
|
WarningsFile string `json:"warnings_file"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type exampleOutputIndexEntry struct {
|
||||||
|
LaneID string `json:"lane_id"`
|
||||||
|
MediaType string `json:"media_type"`
|
||||||
|
File string `json:"file"`
|
||||||
|
SchemaID string `json:"schema_id"`
|
||||||
|
SchemaVersion string `json:"schema_version"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveInputForMaintainedExample(components productionComponents, pipelineID string) config.ResolveInput {
|
||||||
|
return config.ResolveInput{PipelineID: pipelineID, Catalog: catalogFromRegistries(components.registries)}
|
||||||
|
}
|
||||||
442
internal/cli/production_contract_test.go
Normal file
442
internal/cli/production_contract_test.go
Normal file
@@ -0,0 +1,442 @@
|
|||||||
|
package cli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/core/config"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/chunk/scenes"
|
||||||
|
spellcodec "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/codec/spells"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/extract/spells"
|
||||||
|
"gitea.maximumdirect.net/eric/notarius/internal/modules/generic/normalize/noop"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProductionCatalogCoversMaintainedConfigurations(t *testing.T) {
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
registries := components.registries
|
||||||
|
|
||||||
|
assertProductionKeys(t, "inputs", registries.Inputs.RegisteredKeys(), []string{"seriatim"})
|
||||||
|
assertProductionKeys(t, "chunkers", registries.Chunkers.RegisteredKeys(), []string{"dnd/scenes", "generic"})
|
||||||
|
assertProductionKeys(t, "extractors", registries.Extractors.RegisteredKeys(), []string{"dnd/spells"})
|
||||||
|
assertProductionKeys(t, "mergers", registries.Mergers.RegisteredKeys(), []string{"appendorder"})
|
||||||
|
assertProductionKeys(t, "normalizers", registries.Normalizers.RegisteredKeys(), []string{"noop"})
|
||||||
|
assertProductionKeys(t, "outputs", registries.Outputs.RegisteredKeys(), []string{"json"})
|
||||||
|
assertProductionKeys(t, "validators", registries.Validators.RegisteredKeys(), []string{
|
||||||
|
"extract/dnd/spells/shape",
|
||||||
|
"extract/dnd/spells/source_refs",
|
||||||
|
"extract/dnd/spells/source_relatedness",
|
||||||
|
"generic/always_accept",
|
||||||
|
"generic/always_reject",
|
||||||
|
"generic/valid_json",
|
||||||
|
"generic/valid_json_schema",
|
||||||
|
})
|
||||||
|
if got := registries.ArtifactCodecs.RegisteredKinds(); !reflect.DeepEqual(got, []contracts.ArtifactKind{dnd.SpellListKind}) {
|
||||||
|
t.Fatalf("artifact codec kinds = %#v, want [%q]", got, dnd.SpellListKind)
|
||||||
|
}
|
||||||
|
if got := registries.Mergers.RegisteredArtifactKinds(pipeline.DefaultMergeModule); !reflect.DeepEqual(got, []contracts.ArtifactKind{dnd.SpellListKind}) {
|
||||||
|
t.Fatalf("merger variants = %#v, want [%q]", got, dnd.SpellListKind)
|
||||||
|
}
|
||||||
|
if got := registries.Normalizers.RegisteredArtifactKinds(pipeline.DefaultNormalizeModule); !reflect.DeepEqual(got, []contracts.ArtifactKind{dnd.SpellListKind}) {
|
||||||
|
t.Fatalf("normalizer variants = %#v, want [%q]", got, dnd.SpellListKind)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantChain := []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding("generic/valid_json"),
|
||||||
|
pipeline.Binding("generic/valid_json_schema"),
|
||||||
|
pipeline.Binding("extract/dnd/spells/shape"),
|
||||||
|
pipeline.Binding("extract/dnd/spells/source_refs"),
|
||||||
|
pipeline.Binding("extract/dnd/spells/source_relatedness"),
|
||||||
|
}
|
||||||
|
if got := registries.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
|
t.Fatalf("spell validator chain = %#v, want %#v", got, wantChain)
|
||||||
|
}
|
||||||
|
|
||||||
|
assetNames := productionAssetNames(t, components.assets.PromptFS)
|
||||||
|
wantAssets := []string{
|
||||||
|
"dnd.scenes/dnd.scenes.yaml",
|
||||||
|
"dnd.scenes/instructions.md",
|
||||||
|
"dnd.scenes/sharedassets/common-dnd-references.md",
|
||||||
|
"dnd.scenes/sharedassets/common-dnd-system.md",
|
||||||
|
"dnd.scenes/sharedassets/common-dnd-transcript.md",
|
||||||
|
"dnd.scenes/task.md",
|
||||||
|
"dnd.spells/dnd.spells.yaml",
|
||||||
|
"dnd.spells/instructions.md",
|
||||||
|
"dnd.spells/sharedassets/common-dnd-references.md",
|
||||||
|
"dnd.spells/sharedassets/common-dnd-system.md",
|
||||||
|
"dnd.spells/sharedassets/common-dnd-transcript.md",
|
||||||
|
"dnd.spells/task.md",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(assetNames, wantAssets) {
|
||||||
|
t.Fatalf("production prompt assets = %#v, want %#v", assetNames, wantAssets)
|
||||||
|
}
|
||||||
|
|
||||||
|
catalog := catalogFromRegistries(registries)
|
||||||
|
converted := registriesFromCatalog(catalog)
|
||||||
|
if converted.ArtifactCodecs != registries.ArtifactCodecs || converted.ValidatorChains != registries.ValidatorChains {
|
||||||
|
t.Fatal("catalog/registry conversion did not preserve codec and validator-chain registries")
|
||||||
|
}
|
||||||
|
codecSpec, ok := catalog.ArtifactCodecs.Spec(dnd.SpellListKind)
|
||||||
|
if !ok || codecSpec.Kind != dnd.SpellListKind || codecSpec.Schema.ID != spellcodec.SchemaID {
|
||||||
|
t.Fatalf("catalog codec spec = %#v, ok=%t, want typed D&D spell codec", codecSpec, ok)
|
||||||
|
}
|
||||||
|
if got := catalog.ValidatorChains.Validators(pipeline.StageExtract, spells.Key); !reflect.DeepEqual(got, wantChain) {
|
||||||
|
t.Fatalf("catalog validator chain = %#v, want %#v", got, wantChain)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
|
cfg := loadMaintainedExample(t, example.path)
|
||||||
|
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-session", Catalog: catalog})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve %s: %v", example.name, err)
|
||||||
|
}
|
||||||
|
if effective.ResolvedPipeline.Input.Module != "seriatim" || len(effective.ResolvedPipeline.ArtifactLanes) != 1 || effective.ResolvedPipeline.ArtifactLanes[0].ID != "spells" {
|
||||||
|
t.Fatalf("resolved %s pipeline = %#v, want seriatim and spells", example.name, effective.ResolvedPipeline)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProductionPromptAssetsPrepareWithoutProviderCredentials(t *testing.T) {
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
cfg := config.Default()
|
||||||
|
cfg.Pipelines["dnd-scenes"] = pipeline.PipelineProfile{
|
||||||
|
ID: "dnd-scenes",
|
||||||
|
Input: pipeline.Binding("seriatim"),
|
||||||
|
Chunk: pipeline.Binding("dnd/scenes"),
|
||||||
|
Artifacts: map[string]pipeline.ArtifactLaneProfile{
|
||||||
|
"spells": {Extract: pipeline.Binding("dnd/spells")},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
effective, err := cfg.Resolve(config.ResolveInput{PipelineID: "dnd-scenes", Catalog: catalogFromRegistries(components.registries)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve production scene pipeline: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := pipeline.Prepare(effective.ResolvedPipeline, components.registries, pipeline.ModuleDependencies{LLM: &productionFakeLLMClient{}}); err != nil {
|
||||||
|
t.Fatalf("prepare production scene and spell modules: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProductionConfigValidationCoversModuleAndVariantFailures(t *testing.T) {
|
||||||
|
base := string(readRepositoryFile(t, "examples", "dnd-spells.config.yml"))
|
||||||
|
validPath := writeProductionContractConfig(t, base)
|
||||||
|
options := productionCLIOptions(t)
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
if code := RunWithOptions([]string{"config", "validate", "--config", validPath, "--pipeline", "dnd-session"}, &stdout, &stderr, options); code != 0 {
|
||||||
|
t.Fatalf("valid production config: code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
for _, example := range maintainedExampleFiles(t) {
|
||||||
|
var exampleStdout, exampleStderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", example.path, "--pipeline", "dnd-session"}, &exampleStdout, &exampleStderr, options)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("validate maintained %s config: code=%d stdout=%q stderr=%q", example.name, code, exampleStdout.String(), exampleStderr.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
options Options
|
||||||
|
fragments []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "unknown module",
|
||||||
|
content: strings.Replace(base, " input: seriatim\n", " input: missing/input\n", 1),
|
||||||
|
options: productionCLIOptions(t),
|
||||||
|
fragments: []string{"pipeline \"dnd-session\"", "input", "missing/input"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "unknown validator",
|
||||||
|
content: strings.Replace(base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: missing/validator\n", 1),
|
||||||
|
options: productionCLIOptions(t),
|
||||||
|
fragments: []string{"validator", "missing/validator"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid artifact variant",
|
||||||
|
content: base,
|
||||||
|
options: productionCLIOptionsWithoutSpellNormalizer(t),
|
||||||
|
fragments: []string{"normalizer", "noop", string(dnd.SpellListKind), "variant"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deterministic validator with profile",
|
||||||
|
content: strings.Replace(base, " extract: dnd/spells\n", " extract:\n module: dnd/spells\n validators:\n - module: generic/valid_json\n llm_profile: forbidden-profile\n", 1),
|
||||||
|
options: productionCLIOptions(t),
|
||||||
|
fragments: []string{"deterministic validator", "llm_profile"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
path := writeProductionContractConfig(t, tt.content)
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{"config", "validate", "--config", path, "--pipeline", "dnd-session"}, &stdout, &stderr, tt.options)
|
||||||
|
if code != 1 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
for _, fragment := range tt.fragments {
|
||||||
|
if !strings.Contains(stderr.String(), fragment) {
|
||||||
|
t.Fatalf("stderr=%q, want %q", stderr.String(), fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProductionSceneRunRecordsChunkerWarningsAndProvenance(t *testing.T) {
|
||||||
|
outputRoot := filepath.Join(t.TempDir(), "output")
|
||||||
|
configPath := writeProductionContractConfig(t, productionRunConfig(outputRoot, "dnd/scenes"))
|
||||||
|
fake := &productionFakeLLMClient{}
|
||||||
|
options := productionRunOptions(t, fake)
|
||||||
|
var stdout, stderr strings.Builder
|
||||||
|
code := RunWithOptions([]string{
|
||||||
|
"run", "dnd-session", "--config", configPath,
|
||||||
|
"--input", repositoryPath("examples", "seriatim-minimal-transcript.json"),
|
||||||
|
"--chunk_cache", "bypass", "--session-id", "offline-session",
|
||||||
|
}, &stdout, &stderr, options)
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout.String(), stderr.String())
|
||||||
|
}
|
||||||
|
manifest := readProductionJSON[artifacts.RunManifest](t, filepath.Join(outputRoot, productionRunID, "manifest.json"))
|
||||||
|
if manifest.Chunker != scenes.Key || manifest.ChunkPlan == nil || manifest.ChunkPlan.Action != "bypassed" || manifest.ChunkPlan.ProducerModule != scenes.Key {
|
||||||
|
t.Fatalf("chunk manifest = %#v, want dnd scene producer", manifest.ChunkPlan)
|
||||||
|
}
|
||||||
|
if got := manifest.ModuleMetadata["chunker"]["prompt_id"]; got != scenes.PromptID {
|
||||||
|
t.Fatalf("chunker prompt metadata = %#v, want %q", got, scenes.PromptID)
|
||||||
|
}
|
||||||
|
if got := manifest.ChunkPlan.ProducerMetadata["response_schema_id"]; got != scenes.ResponseSchemaID {
|
||||||
|
t.Fatalf("chunk producer schema metadata = %#v, want %q", got, scenes.ResponseSchemaID)
|
||||||
|
}
|
||||||
|
warnings := readProductionJSON[struct {
|
||||||
|
Warnings []contracts.Warning `json:"warnings"`
|
||||||
|
}](t, filepath.Join(outputRoot, productionRunID, "warnings.json"))
|
||||||
|
if len(warnings.Warnings) != 1 || warnings.Warnings[0].ReasonCode != "scene_boundary_caveat" {
|
||||||
|
t.Fatalf("warnings = %#v, want one scene boundary warning", warnings.Warnings)
|
||||||
|
}
|
||||||
|
if len(fake.requestsFor(scenes.PromptID)) != 1 || len(fake.requestsFor(spells.PromptID)) != 1 {
|
||||||
|
t.Fatalf("fake prompt requests = %#v, want one scene and one spell request", fake.requestPrompts())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type maintainedExample struct {
|
||||||
|
name string
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
func maintainedExampleFiles(t *testing.T) []maintainedExample {
|
||||||
|
t.Helper()
|
||||||
|
return []maintainedExample{
|
||||||
|
{name: "minimal", path: repositoryPath("examples", "dnd-spells.config.yml")},
|
||||||
|
{name: "production", path: repositoryPath("examples", "dnd-spells-production.config.yml")},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadMaintainedExample(t *testing.T, path string) config.Config {
|
||||||
|
t.Helper()
|
||||||
|
fileConfig, err := config.LoadFileConfig(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load maintained config %q: %v", path, err)
|
||||||
|
}
|
||||||
|
cfg := config.Default()
|
||||||
|
if err := cfg.ApplyFileConfig(fileConfig); err != nil {
|
||||||
|
t.Fatalf("apply maintained config %q: %v", path, err)
|
||||||
|
}
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("validate maintained config %q: %v", path, err)
|
||||||
|
}
|
||||||
|
return cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionTestComponents(t *testing.T) productionComponents {
|
||||||
|
t.Helper()
|
||||||
|
components, err := newProductionComponents()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new production components: %v", err)
|
||||||
|
}
|
||||||
|
return components
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionCLIOptions(t *testing.T) Options {
|
||||||
|
t.Helper()
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
return productionOptionsFromComponents(components)
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionOptionsFromComponents(components productionComponents) Options {
|
||||||
|
return Options{
|
||||||
|
Catalog: catalogFromRegistries(components.registries),
|
||||||
|
Registries: components.registries,
|
||||||
|
LookupEnv: emptyLookup,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionCLIOptionsWithoutSpellNormalizer(t *testing.T) Options {
|
||||||
|
t.Helper()
|
||||||
|
components := productionTestComponents(t)
|
||||||
|
registries := components.registries
|
||||||
|
registries.Normalizers = pipeline.NewNormalizerRegistry()
|
||||||
|
if err := noop.RegisterTyped[dnd.SpellList](registries.Normalizers, contracts.ArtifactKind("test/other")); err != nil {
|
||||||
|
t.Fatalf("register mismatched normalizer: %v", err)
|
||||||
|
}
|
||||||
|
return productionOptionsFromComponents(productionComponents{registries: registries, assets: components.assets})
|
||||||
|
}
|
||||||
|
|
||||||
|
const productionRunID = "run-1700000000000000000-0123456789abcdef0123456789abcdef"
|
||||||
|
|
||||||
|
func productionRunOptions(t *testing.T, fake *productionFakeLLMClient) Options {
|
||||||
|
t.Helper()
|
||||||
|
options := productionCLIOptions(t)
|
||||||
|
options.Now = func() time.Time { return time.Unix(1700000000, 0).UTC() }
|
||||||
|
options.RunIDGenerator = func(time.Time) (string, error) { return productionRunID, nil }
|
||||||
|
options.UserCacheDir = func() (string, error) { return "", errors.New("user cache must not be used") }
|
||||||
|
options.LLMClientFactory = func(context.Context, config.Config, string) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error) {
|
||||||
|
return fake, nil, nil
|
||||||
|
}
|
||||||
|
return options
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionRunConfig(outputRoot, chunkModule string) string {
|
||||||
|
return fmt.Sprintf(`version: 3
|
||||||
|
output:
|
||||||
|
directory: %q
|
||||||
|
cache:
|
||||||
|
chunk_plans:
|
||||||
|
mode: bypass
|
||||||
|
checkpoints: {}
|
||||||
|
debug:
|
||||||
|
directory: %q
|
||||||
|
pipelines:
|
||||||
|
dnd-session:
|
||||||
|
input: seriatim
|
||||||
|
chunk: %s
|
||||||
|
artifacts:
|
||||||
|
spells:
|
||||||
|
extract: dnd/spells
|
||||||
|
`, outputRoot, filepath.Join(filepath.Dir(outputRoot), "debug"), chunkModule)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeProductionContractConfig(t *testing.T, content string) string {
|
||||||
|
t.Helper()
|
||||||
|
path := filepath.Join(t.TempDir(), "config.yml")
|
||||||
|
if err := os.WriteFile(path, []byte(content), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
}
|
||||||
|
|
||||||
|
func productionAssetNames(t *testing.T, getFS func() (fs.FS, error)) []string {
|
||||||
|
t.Helper()
|
||||||
|
fileSystem, err := getFS()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("load production prompt assets: %v", err)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
if err := fs.WalkDir(fileSystem, ".", func(path string, entry fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !entry.IsDir() {
|
||||||
|
names = append(names, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("walk production prompt assets: %v", err)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertProductionKeys(t *testing.T, name string, got, want []string) {
|
||||||
|
t.Helper()
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("%s = %#v, want %#v", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func readProductionJSON[T any](t *testing.T, path string) T {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s: %v", path, err)
|
||||||
|
}
|
||||||
|
var value T
|
||||||
|
if err := json.Unmarshal(data, &value); err != nil {
|
||||||
|
t.Fatalf("decode %s: %v", path, err)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
type productionFakeLLMClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
requests []contracts.StructuredCompletionRequest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *productionFakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
var content []byte
|
||||||
|
switch req.PromptID {
|
||||||
|
case scenes.PromptID:
|
||||||
|
content = []byte(`{"scenes":[{"start_unit_id":1,"end_unit_id":2,"short_title":"Opening scene","primary_mode":"Narrative","main_participants":["Aria"],"summary":"The session opens.","boundary_note":"The opening covers the available transcript.","boundary_confidence":"High"}],"boundary_caveats":["The opening boundary is inferred from the short transcript."]}`)
|
||||||
|
case spells.PromptID:
|
||||||
|
content = []byte(`{"spell_casts":[{"caster":"Aria","spell":"Cure Wounds","effect":"Heals an injured ally.","narrative_description":"Aria restores the fighter after the fight.","source_refs":[{"source_id":"session-alpha","start_unit_id":1,"end_unit_id":1}]}]}`)
|
||||||
|
default:
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("unexpected prompt %q", req.PromptID)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(content, out); err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("populate fake structured target: %w", err)
|
||||||
|
}
|
||||||
|
client.mu.Lock()
|
||||||
|
client.requests = append(client.requests, req)
|
||||||
|
client.mu.Unlock()
|
||||||
|
return contracts.StructuredCompletionResponse{Content: content, Provider: "test", Model: "deterministic", ProfileID: req.ProfileID}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *productionFakeLLMClient) requestsFor(promptID string) []contracts.StructuredCompletionRequest {
|
||||||
|
client.mu.Lock()
|
||||||
|
defer client.mu.Unlock()
|
||||||
|
var requests []contracts.StructuredCompletionRequest
|
||||||
|
for _, req := range client.requests {
|
||||||
|
if req.PromptID == promptID {
|
||||||
|
requests = append(requests, req)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return requests
|
||||||
|
}
|
||||||
|
|
||||||
|
func (client *productionFakeLLMClient) requestPrompts() []string {
|
||||||
|
client.mu.Lock()
|
||||||
|
defer client.mu.Unlock()
|
||||||
|
prompts := make([]string, 0, len(client.requests))
|
||||||
|
for _, req := range client.requests {
|
||||||
|
prompts = append(prompts, req.PromptID)
|
||||||
|
}
|
||||||
|
return prompts
|
||||||
|
}
|
||||||
|
|
||||||
|
func repositoryPath(parts ...string) string {
|
||||||
|
_, file, _, _ := runtime.Caller(0)
|
||||||
|
return filepath.Join(append([]string{filepath.Dir(file), "..", ".."}, parts...)...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readRepositoryFile(t *testing.T, parts ...string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
data, err := os.ReadFile(repositoryPath(parts...))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user