Verify D&D scene chunker run output
This commit is contained in:
@@ -1062,6 +1062,50 @@ func TestExampleFixtureRunWritesExpectedJSON(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestExampleFixtureRunWithDNDScenesRecordsChunkerAndWarnings(t *testing.T) {
|
||||||
|
configPath := writeTestConfig(t, mvpConfigYAMLWithChunk("dnd-session", scenes.Key, "dnd/spells"))
|
||||||
|
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||||
|
outputDir := t.TempDir()
|
||||||
|
diagnosticsDir := t.TempDir()
|
||||||
|
client := newSceneRunLLMClient("Scene boundary was ambiguous.")
|
||||||
|
var stdout bytes.Buffer
|
||||||
|
var stderr bytes.Buffer
|
||||||
|
|
||||||
|
code := RunWithOptions([]string{"run", "dnd-session", "--config", configPath, "--input", inputPath, "--output-dir", outputDir, "--diagnostics-dir", diagnosticsDir}, &stdout, &stderr, Options{
|
||||||
|
LLMClientFactory: fakeLLMFactory(client, nil),
|
||||||
|
})
|
||||||
|
|
||||||
|
if code != 0 {
|
||||||
|
t.Fatalf("RunWithOptions() code = %d, stderr=%q", code, stderr.String())
|
||||||
|
}
|
||||||
|
if client.calls != 2 {
|
||||||
|
t.Fatalf("LLM calls = %d, want chunking and extraction calls", client.calls)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stderr.String(), "1 warning") {
|
||||||
|
t.Fatalf("stderr = %q, want warning count", stderr.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
runOutputDir := onlyChildDir(t, outputDir)
|
||||||
|
manifestBytes := readFile(t, filepath.Join(runOutputDir, "manifest.json"))
|
||||||
|
var manifest artifacts.RunManifest
|
||||||
|
if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
|
||||||
|
t.Fatalf("unmarshal manifest: %v", err)
|
||||||
|
}
|
||||||
|
if manifest.Chunker != scenes.Key {
|
||||||
|
t.Fatalf("manifest chunker = %q, want %q", manifest.Chunker, scenes.Key)
|
||||||
|
}
|
||||||
|
for _, forbidden := range []string{"Source document ID:", "Aria casts Cure Wounds.", "spell_casts", "Scene boundary was ambiguous."} {
|
||||||
|
if strings.Contains(string(manifestBytes), forbidden) {
|
||||||
|
t.Fatalf("manifest leaked %q: %s", forbidden, manifestBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
warnings := string(readFile(t, filepath.Join(runOutputDir, "warnings.json")))
|
||||||
|
if !strings.Contains(warnings, "scene_boundary_caveat") || !strings.Contains(warnings, "Scene boundary was ambiguous.") {
|
||||||
|
t.Fatalf("warnings output = %s, want scene boundary caveat", warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExampleFixtureRunOnlySpells(t *testing.T) {
|
func TestExampleFixtureRunOnlySpells(t *testing.T) {
|
||||||
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
configPath := fixturePath(t, "examples/dnd-spells.config.yml")
|
||||||
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
inputPath := fixturePath(t, "examples/seriatim-minimal-transcript.json")
|
||||||
@@ -1320,6 +1364,7 @@ type fakeRunLLMClient struct {
|
|||||||
calls int
|
calls int
|
||||||
err error
|
err error
|
||||||
payload map[string]any
|
payload map[string]any
|
||||||
|
sceneCaveat string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
|
func newFakeRunLLMClient(invalidSourceRef bool) *fakeRunLLMClient {
|
||||||
@@ -1334,11 +1379,43 @@ func newMalformedRunLLMClient() *fakeRunLLMClient {
|
|||||||
return &fakeRunLLMClient{payload: map[string]any{}}
|
return &fakeRunLLMClient{payload: map[string]any{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newSceneRunLLMClient(caveat string) *fakeRunLLMClient {
|
||||||
|
return &fakeRunLLMClient{sceneCaveat: caveat}
|
||||||
|
}
|
||||||
|
|
||||||
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
func (client *fakeRunLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||||
client.calls++
|
client.calls++
|
||||||
if client.err != nil {
|
if client.err != nil {
|
||||||
return contracts.StructuredCompletionResponse{}, client.err
|
return contracts.StructuredCompletionResponse{}, client.err
|
||||||
}
|
}
|
||||||
|
if req.StageName == scenes.Key && client.payload == nil {
|
||||||
|
payload := map[string]any{
|
||||||
|
"scenes": []map[string]any{
|
||||||
|
{
|
||||||
|
"start_unit_id": "seg-001",
|
||||||
|
"end_unit_id": "seg-002",
|
||||||
|
"short_title": "Opening spell",
|
||||||
|
"primary_mode": "Narrative",
|
||||||
|
"main_participants": []string{"Aria"},
|
||||||
|
"summary": "Aria casts a spell.",
|
||||||
|
"boundary_note": "The provided source units form one scene.",
|
||||||
|
"boundary_confidence": "High",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"boundary_caveats": []string{},
|
||||||
|
}
|
||||||
|
if client.sceneCaveat != "" {
|
||||||
|
payload["boundary_caveats"] = []string{client.sceneCaveat}
|
||||||
|
}
|
||||||
|
encoded, err := json.Marshal(payload)
|
||||||
|
if err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(encoded, out); err != nil {
|
||||||
|
return contracts.StructuredCompletionResponse{}, err
|
||||||
|
}
|
||||||
|
return contracts.StructuredCompletionResponse{Content: encoded}, nil
|
||||||
|
}
|
||||||
startUnitID := "seg-001"
|
startUnitID := "seg-001"
|
||||||
if client.invalidSourceRef {
|
if client.invalidSourceRef {
|
||||||
startUnitID = "missing-segment"
|
startUnitID = "missing-segment"
|
||||||
|
|||||||
Reference in New Issue
Block a user