From dc7c0e2f9e825018cee3a3fe7eea6617c76ea703 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Fri, 24 Jul 2026 14:40:54 +0000 Subject: [PATCH] Move scene semantics to deterministic validation --- .../schemas/dnd_scene_descriptions_llm.v1.json | 9 +++------ .../extract/scenedescriptions/extractor_test.go | 15 +++++++++++---- .../dnd/extract/scenedescriptions/schema_test.go | 12 +++++++----- .../scenedescriptions/shape/validator_test.go | 4 ++++ 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json b/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json index 64beae5..20ff465 100644 --- a/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json +++ b/internal/modules/dnd/extract/scenedescriptions/assets/schemas/dnd_scene_descriptions_llm.v1.json @@ -6,16 +6,13 @@ "required": ["kind", "title", "summary"], "properties": { "kind": { - "type": "string", - "enum": ["combat", "narrative", "recap", "meta"] + "type": "string" }, "title": { - "type": "string", - "minLength": 1 + "type": "string" }, "summary": { - "type": "string", - "minLength": 1 + "type": "string" } } } diff --git a/internal/modules/dnd/extract/scenedescriptions/extractor_test.go b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go index 57fec1f..3ce7326 100644 --- a/internal/modules/dnd/extract/scenedescriptions/extractor_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/extractor_test.go @@ -70,17 +70,24 @@ func TestExtractPassesOptionalReferencesAndUsesEmptyPlaceholders(t *testing.T) { } } -func TestExtractPreservesTheModelKindWithoutRepair(t *testing.T) { +func TestExtractReturnsSemanticallyInvalidResponseForDeterministicValidation(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatalf("loadResponseSchema() error = %v", err) + } + if err := validateJSONSchema(t, map[string]any{"kind": "unrecognized", "title": " ", "summary": ""}, schema.JSONSchema); err != nil { + t.Fatalf("semantic candidate rejected by private schema: %v", err) + } client := &fakeSceneDescriptionsLLMClient{response: extractionResponse{ - Kind: dnd.SceneKind("unrecognized"), Title: " Untitled scene ", Summary: " Summary ", + Kind: dnd.SceneKind("unrecognized"), Title: " ", Summary: "", }} result, err := newExtractor(t, client).Extract(context.Background(), extractionRequest()) if err != nil { t.Fatalf("Extract() error = %v, want nil", err) } scene := result.Value.Scenes[0] - if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "Untitled scene" || scene.Summary != "Summary" { - t.Fatalf("scene = %#v, want model kind preserved and textual fields trimmed", scene) + if scene.Kind != dnd.SceneKind("unrecognized") || scene.Title != "" || scene.Summary != "" { + t.Fatalf("scene = %#v, want semantic candidates returned for deterministic validation", scene) } } diff --git a/internal/modules/dnd/extract/scenedescriptions/schema_test.go b/internal/modules/dnd/extract/scenedescriptions/schema_test.go index 42b5130..756fd07 100644 --- a/internal/modules/dnd/extract/scenedescriptions/schema_test.go +++ b/internal/modules/dnd/extract/scenedescriptions/schema_test.go @@ -30,10 +30,12 @@ func TestLoadResponseSchemaUsesStrictPrivateSceneDescriptionContract(t *testing. {name: "unknown framework field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "id": "assigned-later"}}, {name: "unknown application field", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "source_ref": map[string]any{}}}, {name: "collection is not allowed", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": "Bandits strike.", "scenes": []any{}}}, - {name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}}, - {name: "wrong field type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}}, - {name: "blank title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}}, - {name: "blank summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}}, + {name: "unsupported kind", response: map[string]any{"kind": "interlude", "title": "Ambush", "summary": "Bandits strike."}, valid: true}, + {name: "empty title", response: map[string]any{"kind": "combat", "title": "", "summary": "Bandits strike."}, valid: true}, + {name: "empty summary", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": ""}, valid: true}, + {name: "wrong kind type", response: map[string]any{"kind": 7, "title": "Ambush", "summary": "Bandits strike."}}, + {name: "wrong title type", response: map[string]any{"kind": "combat", "title": 7, "summary": "Bandits strike."}}, + {name: "wrong summary type", response: map[string]any{"kind": "combat", "title": "Ambush", "summary": 7}}, } { t.Run(test.name, func(t *testing.T) { err := validateJSONSchema(t, test.response, schema.JSONSchema) @@ -54,7 +56,7 @@ func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { } first.JSONSchema[0] = '[' second, err := loadResponseSchema() - if err != nil || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { + if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) } if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil { diff --git a/internal/modules/dnd/validate/scenedescriptions/shape/validator_test.go b/internal/modules/dnd/validate/scenedescriptions/shape/validator_test.go index 75a8bb2..43e9e7e 100644 --- a/internal/modules/dnd/validate/scenedescriptions/shape/validator_test.go +++ b/internal/modules/dnd/validate/scenedescriptions/shape/validator_test.go @@ -27,6 +27,10 @@ func TestValidatorRequiresSceneShapeAndExactlyOneExtractionRecord(t *testing.T) {name: "empty list", value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{}}}, {name: "blank ID", value: list(scene(" ", dnd.SceneKindNarrative, "Arrival", "The party arrives.", 1, 1))}, {name: "unsupported kind", value: list(scene("one", "other", "Arrival", "The party arrives.", 1, 1))}, + {name: "empty title", value: list(scene("one", dnd.SceneKindNarrative, "", "The party arrives.", 1, 1))}, + {name: "whitespace-only title", value: list(scene("one", dnd.SceneKindNarrative, " ", "The party arrives.", 1, 1))}, + {name: "empty summary", value: list(scene("one", dnd.SceneKindNarrative, "Arrival", "", 1, 1))}, + {name: "whitespace-only summary", value: list(scene("one", dnd.SceneKindNarrative, "Arrival", " ", 1, 1))}, {name: "untrimmed prose", value: list(scene("one", dnd.SceneKindNarrative, " Arrival ", "The party arrives.", 1, 1))}, {name: "empty source reference", value: dnd.SceneDescriptionList{Scenes: []dnd.SceneDescription{{ID: "one", Kind: dnd.SceneKindNarrative, Title: "Arrival", Summary: "The party arrives."}}}}, } {