diff --git a/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/instructions.md b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/instructions.md new file mode 100644 index 00000000..4d28fa2b --- /dev/null +++ b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/instructions.md @@ -0,0 +1,11 @@ +Review only whether the proposed scene kind correctly represents substantive +active combat in the supplied transcript chunk under the shared combat policy. +Do not judge the scene title, summary, non-combat subtype, scene boundary, or +any other aspect of a scene description. + +Return `approved` only when the proposed kind's combat status is supported; +approval does not endorse any other part of the scene description. Return +`combat_should_be_added` when a proposed non-combat kind omits substantive +active combat. Return `combat_should_be_removed` when a proposed `combat` kind +is unsupported. Give a concise, transcript-grounded explanation for every +verdict. diff --git a/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/prompt.yaml b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/prompt.yaml new file mode 100644 index 00000000..16f8d0ac --- /dev/null +++ b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/prompt.yaml @@ -0,0 +1,30 @@ +id: dnd.scene_descriptions.validate_combat +version: "v1" +default_profile: dnd-extraction +inputs: + - name: transcript + required: true + content_type: application/json + - name: proposed_kind + required: true + content_type: text/plain +messages: + - role: system + content_file: ./sharedassets/common-dnd-system.md + - role: user + content_file: ./sharedassets/common-dnd-scene-combat-policy.md + - role: user + content_file: ./instructions.md + - role: user + content_file: ./proposed-kind.md + cache_control: + type: ephemeral + - role: user + content_file: ./sharedassets/common-dnd-transcript-chunk.md + cache_control: + type: ephemeral +output: + format: json + validation_mode: json_schema + schema_path: dnd_scene_combat_semantics_llm.v1.json + repair_attempts: 1 diff --git a/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/proposed-kind.md b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/proposed-kind.md new file mode 100644 index 00000000..79aa3487 --- /dev/null +++ b/assets/dnd/scene-descriptions/validate/combat-semantics/prompts/proposed-kind.md @@ -0,0 +1,3 @@ +The proposed scene kind is: + +{{ input "proposed_kind" }} diff --git a/assets/dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json b/assets/dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json new file mode 100644 index 00000000..4581efa3 --- /dev/null +++ b/assets/dnd/scene-descriptions/validate/combat-semantics/schemas/dnd_scene_combat_semantics_llm.v1.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "notarius.dnd.scene_descriptions.combat_semantics.llm", + "type": "object", + "additionalProperties": false, + "required": ["verdict", "explanation"], + "properties": { + "verdict": { + "type": "string", + "enum": ["approved", "combat_should_be_added", "combat_should_be_removed"] + }, + "explanation": { + "type": "string", + "minLength": 1, + "maxLength": 512 + } + } +} diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 46eb439a..6097260c 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -114,6 +114,8 @@ This stage is small enough for one implementation prompt. ## Stage 2: Add Validator Prompt And Schema Assets +✅ Complete + ### Goal Create and locally verify the private PromptKit contract before implementing diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go new file mode 100644 index 00000000..cedbe782 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets.go @@ -0,0 +1,73 @@ +package combat_semantics + +import ( + "fmt" + "io/fs" + "sync" + + rootassets "gitea.maximumdirect.net/eric/notarius/assets" + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/notarius/internal/framework/promptfs" + "gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared" +) + +const promptAssetRoot = "assets/prompts" + +var promptAssetManifest = shared.PromptAssetManifest{ + ModuleDir: PromptID, + ModuleFiles: []promptfs.ModulePromptFile{ + {Name: "prompt.yaml", Path: "prompts/prompt.yaml"}, + {Name: "instructions.md", Path: "prompts/instructions.md"}, + {Name: "proposed-kind.md", Path: "prompts/proposed-kind.md"}, + }, + SharedFiles: []string{ + "common-dnd-system.md", + "common-dnd-scene-combat-policy.md", + "common-dnd-transcript-chunk.md", + }, +} + +func moduleAssetFS() (fs.FS, error) { + assets, err := fs.Sub(rootassets.FS(), "dnd/scene-descriptions/validate/combat-semantics") + if err != nil { + return nil, fmt.Errorf("scope scene combat-semantics validator assets: %w", err) + } + return assets, nil +} + +func RegisterPromptAssets(registry *llm.AssetRegistry) error { + assets, err := moduleAssetFS() + if err != nil { + return err + } + promptFS, err := promptAssetManifest.PromptFS(assets) + if err != nil { + return fmt.Errorf("prepare scene combat-semantics validator prompt assets: %w", err) + } + if err := registry.RegisterPromptFS(promptFS, promptAssetRoot); err != nil { + return err + } + schemas, err := fs.Sub(assets, "schemas") + if err != nil { + return fmt.Errorf("scope scene combat-semantics validator schemas: %w", err) + } + return registry.RegisterSchemaFS(schemas, ".") +} + +func promptAssetMetadata() (string, error) { + promptAssetHashOnce.Do(func() { + assets, err := moduleAssetFS() + if err != nil { + promptAssetHashErr = err + return + } + promptAssetHash, promptAssetHashErr = promptAssetManifest.Hash(assets) + }) + return promptAssetHash, promptAssetHashErr +} + +var ( + promptAssetHashOnce sync.Once + promptAssetHash string + promptAssetHashErr error +) diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go new file mode 100644 index 00000000..514a2fb7 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/prompt_assets_test.go @@ -0,0 +1,100 @@ +package combat_semantics + +import ( + "context" + "strings" + "testing" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestRegisterPromptAssetsPreparesCombatSemanticsPrompt(t *testing.T) { + registry := llm.NewAssetRegistry() + if err := RegisterPromptAssets(registry); err != nil { + t.Fatalf("RegisterPromptAssets() error = %v, want nil", err) + } + engine := newCombatSemanticsPromptEngine(t, registry) + inputs := map[string]promptkit.ArtifactRef{ + "proposed_kind": promptkit.Inline("narrative"), + "transcript": promptkit.InlineWithURI("file:///session.json", `{"sentinel":"combat-semantics-transcript"}`), + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: inputs, + }) + if err != nil { + t.Fatalf("Prepare() error = %v, want nil", err) + } + if prepared.PromptID != PromptID || prepared.OutputContract.SchemaPath != "dnd_scene_combat_semantics_llm.v1.json" { + t.Fatalf("prepared prompt = %#v, want combat-semantics prompt identity and schema wiring", prepared) + } + if len(prepared.Messages) != 5 { + t.Fatalf("prepared message count = %d, want 5", len(prepared.Messages)) + } + for index, message := range prepared.Messages { + wantEphemeral := index == 3 || index == 4 + gotEphemeral := message.CacheControl != nil && message.CacheControl.Type == promptkit.CacheControlEphemeral + if gotEphemeral != wantEphemeral { + t.Fatalf("message %d cache control = %#v, want ephemeral=%t", index, message.CacheControl, wantEphemeral) + } + } + assertRenderedExactlyOnce(t, prepared.Messages, "substantive active combat materially organizes the", 1) + assertRenderedExactlyOnce(t, prepared.Messages, "narrative", 3) + assertRenderedExactlyOnce(t, prepared.Messages, "combat-semantics-transcript", 4) + for name := range inputs { + missing := make(map[string]promptkit.ArtifactRef, len(inputs)-1) + for inputName, input := range inputs { + if inputName != name { + missing[inputName] = input + } + } + if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{ + PromptID: PromptID, PromptVersion: SchemaVersion, ProfileID: "combat-semantics-test-profile", Inputs: missing, + }); err == nil { + t.Fatalf("Prepare() without required %q input = nil error", name) + } + } +} + +func TestPromptAssetMetadataIsContentSafe(t *testing.T) { + hash, err := promptAssetMetadata() + if err != nil || !strings.HasPrefix(hash, "sha256:") { + t.Fatalf("promptAssetMetadata() = %q, %v; want hash", hash, err) + } + if strings.Contains(hash, "combat_should_be_added") { + t.Fatalf("prompt asset metadata leaked raw prompt content: %q", hash) + } +} + +func newCombatSemanticsPromptEngine(t *testing.T, registry *llm.AssetRegistry) *promptkit.Engine { + t.Helper() + options, err := registry.PromptKitOptions() + if err != nil { + t.Fatalf("PromptKitOptions() error = %v, want nil", err) + } + options = append(options, promptkit.WithProfiles(promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ + ID: "combat-semantics-test-profile", Endpoint: "http://127.0.0.1:1/v1", Model: "combat-semantics-test-model", + }))) + engine, err := promptkit.NewEngine(promptkit.Config{Timeout: time.Second}, options...) + if err != nil { + t.Fatalf("NewEngine() error = %v, want nil", err) + } + return engine +} + +func assertRenderedExactlyOnce(t *testing.T, messages []promptkit.RenderedMessage, sentinel string, wantIndex int) { + t.Helper() + index := -1 + occurrences := 0 + for messageIndex, message := range messages { + count := strings.Count(message.Content, sentinel) + if count > 0 { + index = messageIndex + occurrences += count + } + } + if occurrences != 1 || index != wantIndex { + t.Fatalf("sentinel %q rendered %d times at message %d, want once at message %d", sentinel, occurrences, index, wantIndex) + } +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go new file mode 100644 index 00000000..265cda61 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema.go @@ -0,0 +1,25 @@ +package combat_semantics + +import "gitea.maximumdirect.net/eric/notarius/internal/framework/llm" + +const ( + PromptID = "dnd.scene_descriptions.validate_combat" + ResponseSchemaKey = llm.ResponseSchemaKey("dnd_scene_combat_semantics_llm") + ResponseSchemaID = "notarius.dnd.scene_descriptions.combat_semantics.llm" + ResponseSchemaName = "notarius_dnd_scene_combat_semantics_llm_v1" + SchemaVersion = "v1" +) + +func loadResponseSchema() (llm.ResponseSchema, error) { + assets, err := moduleAssetFS() + if err != nil { + return llm.ResponseSchema{}, err + } + return llm.LoadResponseSchema(assets, llm.ResponseSchemaDefinition{ + Key: ResponseSchemaKey, + ID: ResponseSchemaID, + Version: SchemaVersion, + Name: ResponseSchemaName, + AssetPath: "schemas/dnd_scene_combat_semantics_llm.v1.json", + }) +} diff --git a/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go new file mode 100644 index 00000000..b79f0e97 --- /dev/null +++ b/internal/modules/dnd/validate/scenedescriptions/combat_semantics/schema_test.go @@ -0,0 +1,86 @@ +package combat_semantics + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +func TestLoadResponseSchemaUsesStrictCombatSemanticsContract(t *testing.T) { + schema, err := loadResponseSchema() + if err != nil { + t.Fatalf("loadResponseSchema() error = %v, want nil", err) + } + if schema.Key != ResponseSchemaKey || schema.ID != ResponseSchemaID || schema.Version != SchemaVersion || schema.Name != ResponseSchemaName || !strings.HasPrefix(schema.SHA256, "sha256:") || !json.Valid(schema.JSONSchema) { + t.Fatalf("schema = %#v, want private combat-semantics schema identity", schema) + } + for _, verdict := range []string{"approved", "combat_should_be_added", "combat_should_be_removed"} { + if err := validateCombatSemanticsSchema(map[string]any{"verdict": verdict, "explanation": "The chunk contains sustained attack exchanges."}, schema.JSONSchema); err != nil { + t.Fatalf("valid %q verdict rejected: %v", verdict, err) + } + } + for _, test := range []struct { + name string + response map[string]any + }{ + {name: "missing explanation", response: map[string]any{"verdict": "approved"}}, + {name: "missing verdict", response: map[string]any{"explanation": "The proposed kind is supported."}}, + {name: "unknown property", response: map[string]any{"verdict": "approved", "explanation": "The proposed kind is supported.", "confidence": 1}}, + {name: "wrong verdict type", response: map[string]any{"verdict": 1, "explanation": "The proposed kind is supported."}}, + {name: "wrong explanation type", response: map[string]any{"verdict": "approved", "explanation": 1}}, + {name: "unsupported verdict", response: map[string]any{"verdict": "uncertain", "explanation": "The proposed kind is supported."}}, + {name: "blank explanation", response: map[string]any{"verdict": "approved", "explanation": ""}}, + {name: "oversized explanation", response: map[string]any{"verdict": "approved", "explanation": strings.Repeat("a", 513)}}, + } { + t.Run(test.name, func(t *testing.T) { + if err := validateCombatSemanticsSchema(test.response, schema.JSONSchema); err == nil { + t.Fatal("invalid private response accepted") + } + }) + } + if bytes.Contains(schema.JSONSchema, []byte("uniqueItems")) { + t.Fatalf("schema included unsupported uniqueItems constraint: %s", schema.JSONSchema) + } +} + +func TestResponseSchemaIsMutationSafeAndDiagnosticsRedactContent(t *testing.T) { + first, err := loadResponseSchema() + if err != nil { + t.Fatal(err) + } + first.JSONSchema[0] = '[' + second, err := loadResponseSchema() + if err != nil || !strings.HasPrefix(second.SHA256, "sha256:") || !json.Valid(second.JSONSchema) || bytes.Equal(first.JSONSchema, second.JSONSchema) { + t.Fatalf("second schema = %s, %v; want defensive copy", second.JSONSchema, err) + } + if diagnostics := second.DiagnosticsMap(); diagnostics["json_schema"] != nil { + t.Fatalf("schema diagnostics included raw content: %#v", diagnostics) + } +} + +func validateCombatSemanticsSchema(instance map[string]any, schemaContent []byte) error { + instanceContent, err := json.Marshal(instance) + if err != nil { + return err + } + parsedInstance, err := jsonschema.UnmarshalJSON(bytes.NewReader(instanceContent)) + if err != nil { + return err + } + parsedSchema, err := jsonschema.UnmarshalJSON(bytes.NewReader(schemaContent)) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + if err := compiler.AddResource("schema.json", parsedSchema); err != nil { + return err + } + compiled, err := compiler.Compile("schema.json") + if err != nil { + return err + } + return compiled.Validate(parsedInstance) +}