From 6310e49fcea94952e5ef1dc645636515b6fd4922 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 4 Jul 2026 01:15:59 +0000 Subject: [PATCH] Add fixture-driven MVP acceptance coverage --- examples/dnd-spells.config.yml | 16 + examples/seriatim-minimal-transcript.json | 22 ++ internal/cli/run_test.go | 290 +++++++++++++++++- .../invalid-seriatim-empty-segments.json | 6 + .../modules/extract/dnd/spells/runner_test.go | 9 + 5 files changed, 330 insertions(+), 13 deletions(-) create mode 100644 examples/dnd-spells.config.yml create mode 100644 examples/seriatim-minimal-transcript.json create mode 100644 internal/cli/testdata/invalid-seriatim-empty-segments.json diff --git a/examples/dnd-spells.config.yml b/examples/dnd-spells.config.yml new file mode 100644 index 0000000..d66fb97 --- /dev/null +++ b/examples/dnd-spells.config.yml @@ -0,0 +1,16 @@ +version: 1 +llm_profiles: + default: + provider: openai-compatible + base_url: http://127.0.0.1:1 + model: fake-model +pipelines: + dnd-session: + input: seriatim + chunk: + module: generic + options: + max_units: 50 + artifacts: + spells: + extract: dnd/spells diff --git a/examples/seriatim-minimal-transcript.json b/examples/seriatim-minimal-transcript.json new file mode 100644 index 0000000..abf9c26 --- /dev/null +++ b/examples/seriatim-minimal-transcript.json @@ -0,0 +1,22 @@ +{ + "metadata": { + "id": "session-alpha", + "title": "Synthetic D&D spell session" + }, + "segments": [ + { + "id": "seg-001", + "start": 0, + "end": 4, + "speaker": "Aria", + "text": "Aria raises her holy symbol and casts Cure Wounds." + }, + { + "id": "seg-002", + "start": 4, + "end": 8, + "speaker": "DM", + "text": "The bandit mage casts Shield as the blow lands." + } + ] +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 052536d..f80b14c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -900,6 +900,238 @@ func TestRunPipelineDiagnosticsDirFlagOverridesConfig(t *testing.T) { } } +func TestExampleFixtureConfigValidateAndPipelinesList(t *testing.T) { + configPath := fixturePath(t, "examples/dnd-spells.config.yml") + + t.Run("validate configured pipeline", func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions([]string{"config", "validate", "--config", configPath, "--pipeline", "dnd-session"}, &stdout, &stderr, Options{}) + + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + if !strings.Contains(stdout.String(), "dnd-session") { + t.Fatalf("stdout = %q, want pipeline ID", stdout.String()) + } + }) + + t.Run("list configured pipelines", func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions([]string{"pipelines", "list", "--config", configPath}, &stdout, &stderr, Options{}) + + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + if got, want := stdout.String(), "dnd-session\n"; got != want { + t.Fatalf("stdout = %q, want %q", got, want) + } + }) +} + +func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) { + configPath := fixturePath(t, "examples/dnd-spells.config.yml") + inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") + outputDir := t.TempDir() + diagnosticsDir := t.TempDir() + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ + LLMClientFactory: fakeLLMFactory(newFakeRunLLMClient(false), nil), + }) + + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + runOutputDir := onlyChildDir(t, outputDir) + if !strings.Contains(stdout.String(), runOutputDir) { + t.Fatalf("stdout = %q, want output path %q", stdout.String(), runOutputDir) + } + + var manifest artifacts.RunManifest + readJSONFile(t, filepath.Join(runOutputDir, "manifest.json"), &manifest) + if manifest.PipelineID != "dnd-session" { + t.Fatalf("manifest pipeline ID = %q, want dnd-session", manifest.PipelineID) + } + if manifest.PipelineDigest == "" { + t.Fatal("manifest pipeline digest is empty") + } + if manifest.ValidationStatus != "approved" { + t.Fatalf("validation status = %q, want approved", manifest.ValidationStatus) + } + if len(manifest.ArtifactLanes) != 1 { + t.Fatalf("artifact lanes = %#v, want one lane", manifest.ArtifactLanes) + } + extractorMetadata, ok := manifest.ArtifactLanes[0].Metadata["extractor"].(map[string]any) + if !ok { + t.Fatalf("extractor metadata = %#v, want object", manifest.ArtifactLanes[0].Metadata) + } + if extractorMetadata["prompt_id"] != "dnd.spells" || extractorMetadata["response_schema_key"] != "dnd_spells" { + t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata) + } + + var artifactFile struct { + ArtifactType string `json:"artifact_type"` + Artifacts []artifacts.Artifact `json:"artifacts"` + } + readJSONFile(t, filepath.Join(runOutputDir, "artifacts", "dnd.spell_cast.json"), &artifactFile) + if artifactFile.ArtifactType != "dnd.spell_cast" || len(artifactFile.Artifacts) != 1 { + t.Fatalf("artifact file = %#v, want one spell artifact", artifactFile) + } + var payload struct { + Caster string `json:"caster"` + Spell string `json:"spell"` + Effect string `json:"effect"` + } + if err := json.Unmarshal(artifactFile.Artifacts[0].Payload, &payload); err != nil { + t.Fatalf("unmarshal spell payload: %v", err) + } + if payload.Caster != "Aria" || payload.Spell != "Cure Wounds" || payload.Effect == "" { + t.Fatalf("payload = %#v, want deterministic spell output", payload) + } + if len(artifactFile.Artifacts[0].SourceRefs) != 1 { + t.Fatalf("source refs = %#v, want one source ref", artifactFile.Artifacts[0].SourceRefs) + } + ref := artifactFile.Artifacts[0].SourceRefs[0] + if ref.SourceID != "session-alpha" || ref.StartUnitID != "seg-001" || ref.EndUnitID != "seg-001" { + t.Fatalf("source ref = %#v, want fixture source ref", ref) + } + + warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json"))) + if !strings.Contains(warnings, `"warnings": []`) { + t.Fatalf("warnings output = %s, want empty warnings", warnings) + } +} + +func TestExampleFixtureRunOnlySpells(t *testing.T) { + configPath := fixturePath(t, "examples/dnd-spells.config.yml") + inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") + outputDir := t.TempDir() + diagnosticsDir := t.TempDir() + client := newFakeRunLLMClient(false) + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "spells", "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{ + LLMClientFactory: fakeLLMFactory(client, nil), + }) + + if code != 0 { + t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String()) + } + if client.calls != 1 { + t.Fatalf("LLM calls = %d, want selected lane once", client.calls) + } + if _, err := os.Stat(filepath.Join(onlyChildDir(t, outputDir), "manifest.json")); err != nil { + t.Fatalf("expected manifest output: %v", err) + } +} + +func TestExampleFixtureFailureCoverage(t *testing.T) { + configPath := fixturePath(t, "examples/dnd-spells.config.yml") + inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json") + + tests := []struct { + name string + args []string + factory LLMClientFactory + wantCode int + wantStderr string + wantOutputStatus string + }{ + { + name: "missing config", + args: []string{"config", "validate", "--config", filepath.Join(filepath.Dir(configPath), "missing.yml"), "--pipeline", "dnd-session"}, + wantCode: 1, + wantStderr: "config file", + }, + { + name: "unknown pipeline", + args: []string{"run", "missing", "--config", configPath, "--input", inputPath}, + factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), + wantCode: 1, + wantStderr: "not configured", + }, + { + name: "invalid Seriatim input", + args: []string{"run", "dnd-session", "--config", configPath, "--input", fixturePath(t, "internal/cli/testdata/invalid-seriatim-empty-segments.json")}, + factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), + wantCode: 1, + wantStderr: "segments must not be empty", + }, + { + name: "invalid only lane", + args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--only", "missing"}, + factory: fakeLLMFactory(newFakeRunLLMClient(false), nil), + wantCode: 1, + wantStderr: "selected artifact lane", + }, + { + name: "fake LLM failure", + args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, + factory: fakeLLMFactory(newErrorRunLLMClient(errors.New("completion unavailable")), nil), + wantCode: 1, + wantStderr: "completion unavailable", + }, + { + name: "malformed LLM response", + args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, + factory: fakeLLMFactory(newMalformedRunLLMClient(), nil), + wantCode: 1, + wantStderr: "spell_casts", + }, + { + name: "invalid source reference rejection", + args: []string{"run", "dnd-session", "--config", configPath, "--input", inputPath}, + factory: fakeLLMFactory(newFakeRunLLMClient(true), nil), + wantCode: 0, + wantOutputStatus: "rejected", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outputDir := t.TempDir() + diagnosticsDir := t.TempDir() + args := append([]string(nil), test.args...) + if args[0] == "run" { + args = append(args, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir) + } + factory := test.factory + if factory == nil { + factory = fakeLLMFactory(newFakeRunLLMClient(false), nil) + } + var stdout bytes.Buffer + var stderr bytes.Buffer + + code := RunWithOptions(args, &stdout, &stderr, Options{LLMClientFactory: factory}) + + if code != test.wantCode { + t.Fatalf("RunWithOptions() code = %d, want %d; stderr=%q", code, test.wantCode, stderr.String()) + } + if test.wantStderr != "" && !strings.Contains(stderr.String(), test.wantStderr) { + t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.wantStderr) + } + if test.wantOutputStatus != "" { + runOutputDir := onlyChildDir(t, outputDir) + var manifest artifacts.RunManifest + readJSONFile(t, filepath.Join(runOutputDir, "manifest.json"), &manifest) + if manifest.ValidationStatus != test.wantOutputStatus { + t.Fatalf("validation status = %q, want %q", manifest.ValidationStatus, test.wantOutputStatus) + } + rejected := string(readFile(t, filepath.Join(runOutputDir, "rejected.json"))) + if !strings.Contains(rejected, "invalid_source_ref") { + t.Fatalf("rejected output = %s, want invalid source ref rejection", rejected) + } + } + }) + } +} + func writeTestConfig(t *testing.T, content string) string { t.Helper() return writeFile(t, "config.yml", content) @@ -914,6 +1146,15 @@ func writeFile(t *testing.T, name string, content string) string { return path } +func fixturePath(t *testing.T, name string) string { + t.Helper() + path := filepath.Join("..", "..", filepath.FromSlash(name)) + if _, err := os.Stat(path); err != nil { + t.Fatalf("fixture %q is not available at %q: %v", name, path, err) + } + return path +} + func testConfigYAML(pipelineID string, laneIDs ...string) string { return testConfigYAMLForPipelines(map[string][]string{pipelineID: laneIDs}) } @@ -1010,34 +1251,50 @@ func writeSeriatimInput(t *testing.T) string { type fakeRunLLMClient struct { invalidSourceRef bool calls int + err error + payload map[string]any } func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient { return &fakeRunLLMClient{invalidSourceRef: invalidSourceRef} } +func newErrorRunLLMClient(err error) *fakeRunLLMClient { + return &fakeRunLLMClient{err: err} +} + +func newMalformedRunLLMClient() *fakeRunLLMClient { + return &fakeRunLLMClient{payload: map[string]any{}} +} + func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) { client.calls++ + if client.err != nil { + return contracts.StructuredCompletionResponse{}, client.err + } startUnitID := "seg-001" if client.invalidSourceRef { startUnitID = "missing-segment" } - payload := map[string]any{ - "spell_casts": []map[string]any{ - { - "caster": "Aria", - "spell": "Cure Wounds", - "effect": "Heals a wounded ally.", - "narrative_description": "Aria casts Cure Wounds.", - "source_refs": []map[string]string{ - { - "source_id": "session-alpha", - "start_unit_id": startUnitID, - "end_unit_id": "seg-001", + payload := client.payload + if payload == nil { + payload = map[string]any{ + "spell_casts": []map[string]any{ + { + "caster": "Aria", + "spell": "Cure Wounds", + "effect": "Heals a wounded ally.", + "narrative_description": "Aria casts Cure Wounds.", + "source_refs": []map[string]string{ + { + "source_id": "session-alpha", + "start_unit_id": startUnitID, + "end_unit_id": "seg-001", + }, }, }, }, - }, + } } encoded, err := json.Marshal(payload) if err != nil { @@ -1157,6 +1414,13 @@ func readFile(t *testing.T, path string) []byte { return data } +func readJSONFile(t *testing.T, path string, out any) { + t.Helper() + if err := json.Unmarshal(readFile(t, path), out); err != nil { + t.Fatalf("unmarshal %q: %v", path, err) + } +} + func assertNoTemporaryFiles(t *testing.T, root string) { t.Helper() if err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { diff --git a/internal/cli/testdata/invalid-seriatim-empty-segments.json b/internal/cli/testdata/invalid-seriatim-empty-segments.json new file mode 100644 index 0000000..65b3061 --- /dev/null +++ b/internal/cli/testdata/invalid-seriatim-empty-segments.json @@ -0,0 +1,6 @@ +{ + "metadata": { + "id": "session-alpha" + }, + "segments": [] +} diff --git a/internal/modules/extract/dnd/spells/runner_test.go b/internal/modules/extract/dnd/spells/runner_test.go index 74ce0b1..ace6678 100644 --- a/internal/modules/extract/dnd/spells/runner_test.go +++ b/internal/modules/extract/dnd/spells/runner_test.go @@ -93,6 +93,15 @@ func TestRunnerProcessesSeriatimInputWithDNDSpellsExtractor(t *testing.T) { if lane.ID != "spells" || lane.Extractor != Key { t.Fatalf("manifest lane = %#v, want spells lane with dnd/spells extractor", lane) } + extractorMetadata, ok := lane.Metadata["extractor"].(map[string]any) + if !ok { + t.Fatalf("manifest lane metadata = %#v, want extractor metadata", lane.Metadata) + } + if extractorMetadata["prompt_id"] != PromptID || + extractorMetadata["response_schema_key"] != string(ResponseSchemaKey) || + extractorMetadata["response_schema_name"] != ResponseSchemaName { + t.Fatalf("extractor metadata = %#v, want prompt/schema identifiers", extractorMetadata) + } if output.ContentType != "application/json" { t.Fatalf("ContentType = %q, want application/json", output.ContentType) }