Verify D&D scene chunker run output
This commit is contained in:
@@ -1062,6 +1062,50 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
|
||||
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
|
||||
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||
outputDir := t.TempDir()
|
||||
diagnosticsDir := t.TempDir()
|
||||
client := newSceneRunLLMClient("Scene boundary was ambiguous.")
|
||||
var stdout bytes.Buffer
|
||||
var stderr bytes.Buffer
|
||||
|
||||
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||
})
|
||||
|
||||
if code != 0 {
|
||||
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||
}
|
||||
if client.calls != 2 {
|
||||
t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls)
|
||||
}
|
||||
if !strings.Contains(stderr.String(), "1 warning") {
|
||||
t.Fatalf("stderr = %q, want warning count", stderr.String())
|
||||
}
|
||||
|
||||
runOutputDir := onlyChildDir(t, outputDir)
|
||||
manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json"))
|
||||
var manifest artifacts.RunManifest
|
||||
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||
t.Fatalf("unmarshal manifest: %v", err)
|
||||
}
|
||||
if manifest.Chunker != scenes.Key {
|
||||
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
|
||||
}
|
||||
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
|
||||
if strings.Contains(string(manifestBytes), forbidden) {
|
||||
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
|
||||
}
|
||||
}
|
||||
|
||||
warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json")))
|
||||
if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") {
|
||||
t.Fatalf("warnings output = %s, want scene boundary caveat", warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExampleFixtureRunOnlySpells(t *testing.T) {
|
||||
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
||||
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||
@@ -1320,6 +1364,7 @@ type fakeRunLLMClient struct {
|
||||
calls int
|
||||
err error
|
||||
payload map[string]any
|
||||
sceneCaveat string
|
||||
}
|
||||
|
||||
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
|
||||
@@ -1334,11 +1379,43 @@ func newMalformedRunLLMClient() *fakeRunLLMClient {
|
||||
return &fakeRunLLMClient{payload: map[string]any{}}
|
||||
}
|
||||
|
||||
func newSceneRunLLMClient(caveat string) *fakeRunLLMClient {
|
||||
return &fakeRunLLMClient{sceneCaveat: caveat}
|
||||
}
|
||||
|
||||
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
client.calls++
|
||||
if client.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, client.err
|
||||
}
|
||||
if req.StageName == scenes.Key && client.payload == nil {
|
||||
payload := map[string]any{
|
||||
"scenes": []map[string]any{
|
||||
{
|
||||
"start_unit_id": "seg-001",
|
||||
"end_unit_id": "seg-002",
|
||||
"short_title": "Opening spell",
|
||||
"primary_mode": "Narrative",
|
||||
"main_participants": []string{"Aria"},
|
||||
"summary": "Aria casts a spell.",
|
||||
"boundary_note": "The provided source units form one scene.",
|
||||
"boundary_confidence": "High",
|
||||
},
|
||||
},
|
||||
"boundary_caveats": []string{},
|
||||
}
|
||||
if client.sceneCaveat != "" {
|
||||
payload["boundary_caveats"] = []string{client.sceneCaveat}
|
||||
}
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
if err := json.Unmarshal(encoded, out); err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, err
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Content: encoded}, nil
|
||||
}
|
||||
startUnitID := "seg-001"
|
||||
if client.invalidSourceRef {
|
||||
startUnitID = "missing-segment"
|
||||
|
||||
Reference in New Issue
Block a user