Verify D&D scene chunker run output

This commit is contained in:
2026-07-04 13:08:45 +00:00
parent 2130414899
commit c8217549a8

View File

@@ -1062,6 +1062,50 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
}
}
func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
outputDir := t.TempDir()
diagnosticsDir := t.TempDir()
client := newSceneRunLLMClient("Scene boundary was ambiguous.")
var stdout bytes.Buffer
var stderr bytes.Buffer
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
LLMClientFactory: fakeLLMFactory(client, nil),
})
if code != 0 {
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
}
if client.calls != 2 {
t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls)
}
if !strings.Contains(stderr.String(), "1 warning") {
t.Fatalf("stderr = %q, want warning count", stderr.String())
}
runOutputDir := onlyChildDir(t, outputDir)
manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json"))
var manifest artifacts.RunManifest
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
t.Fatalf("unmarshal manifest: %v", err)
}
if manifest.Chunker != scenes.Key {
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
}
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
if strings.Contains(string(manifestBytes), forbidden) {
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
}
}
warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json")))
if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") {
t.Fatalf("warnings output = %s, want scene boundary caveat", warnings)
}
}
func TestExampleFixtureRunOnlySpells(t *testing.T) {
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
@@ -1320,6 +1364,7 @@ type fakeRunLLMClient struct {
calls int
err error
payload map[string]any
sceneCaveat string
}
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
@@ -1334,11 +1379,43 @@ func newMalformedRunLLMClient() *fakeRunLLMClient {
return &fakeRunLLMClient{payload: map[string]any{}}
}
func newSceneRunLLMClient(caveat string) *fakeRunLLMClient {
return &fakeRunLLMClient{sceneCaveat: caveat}
}
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
client.calls++
if client.err != nil {
return contracts.StructuredCompletionResponse{}, client.err
}
if req.StageName == scenes.Key && client.payload == nil {
payload := map[string]any{
"scenes": []map[string]any{
{
"start_unit_id": "seg-001",
"end_unit_id": "seg-002",
"short_title": "Opening spell",
"primary_mode": "Narrative",
"main_participants": []string{"Aria"},
"summary": "Aria casts a spell.",
"boundary_note": "The provided source units form one scene.",
"boundary_confidence": "High",
},
},
"boundary_caveats": []string{},
}
if client.sceneCaveat != "" {
payload["boundary_caveats"] = []string{client.sceneCaveat}
}
encoded, err := json.Marshal(payload)
if err != nil {
return contracts.StructuredCompletionResponse{}, err
}
if err := json.Unmarshal(encoded, out); err != nil {
return contracts.StructuredCompletionResponse{}, err
}
return contracts.StructuredCompletionResponse{Content: encoded}, nil
}
startUnitID := "seg-001"
if client.invalidSourceRef {
startUnitID = "missing-segment"