Implement combat semantics validator
This commit is contained in:
@@ -0,0 +1,233 @@
|
||||
package combat_semantics
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/shared"
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/modules/dnd/validate/scenedescriptions/shape"
|
||||
)
|
||||
|
||||
const (
|
||||
Key = "extract/dnd/scene-descriptions/combat_semantics"
|
||||
ReasonCodeActiveCombatNotClassified = "scene_active_combat_not_classified"
|
||||
ReasonCodeCombatClassificationUnsupported = "scene_combat_classification_unsupported"
|
||||
combatPolicy = "dnd.scene_descriptions.combat_semantics.v1"
|
||||
maximumExplanationRunes = 512
|
||||
)
|
||||
|
||||
type Options struct{}
|
||||
|
||||
type completionResponse struct {
|
||||
Verdict string `json:"verdict"`
|
||||
Explanation string `json:"explanation"`
|
||||
}
|
||||
|
||||
type Validator struct {
|
||||
llm contracts.StructuredLLMClient
|
||||
promptSHA string
|
||||
responseSchemaSHA string
|
||||
}
|
||||
|
||||
var _ contracts.TypedValidator[dnd.SceneDescriptionList] = (*Validator)(nil)
|
||||
var _ contracts.ManifestMetadataProvider = (*Validator)(nil)
|
||||
var _ pipeline.CheckpointFingerprintProvider = (*Validator)(nil)
|
||||
|
||||
func New(llmClient contracts.StructuredLLMClient, _ Options) (*Validator, error) {
|
||||
if llmClient == nil {
|
||||
return nil, validatorErrorf("LLM client must not be nil")
|
||||
}
|
||||
promptSHA, err := promptAssetMetadata()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load prompt metadata: %w", err)
|
||||
}
|
||||
responseSchema, err := loadResponseSchema()
|
||||
if err != nil {
|
||||
return nil, validatorErrorf("load response schema: %w", err)
|
||||
}
|
||||
return &Validator{llm: llmClient, promptSHA: promptSHA, responseSchemaSHA: responseSchema.SHA256}, nil
|
||||
}
|
||||
|
||||
func (v *Validator) Name() string { return Key }
|
||||
|
||||
func (v *Validator) ExecutionClass() contracts.ExecutionClass {
|
||||
return contracts.ExecutionClassLLMBacked
|
||||
}
|
||||
|
||||
func (v *Validator) ManifestMetadata() map[string]any {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{
|
||||
"prompt_id": PromptID,
|
||||
"prompt_version": SchemaVersion,
|
||||
"prompt_sha256": v.promptSHA,
|
||||
"response_schema_key": string(ResponseSchemaKey),
|
||||
"response_schema_id": ResponseSchemaID,
|
||||
"response_schema_name": ResponseSchemaName,
|
||||
"response_schema_version": SchemaVersion,
|
||||
"response_schema_sha256": v.responseSchemaSHA,
|
||||
"combat_policy": combatPolicy,
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) CheckpointFingerprints() []pipeline.CheckpointFingerprint {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return []pipeline.CheckpointFingerprint{
|
||||
{Name: "prompt", Value: v.promptSHA},
|
||||
{Name: "response_schema", Value: v.responseSchemaSHA},
|
||||
{Name: "combat_policy", Value: combatPolicy},
|
||||
}
|
||||
}
|
||||
|
||||
func (v *Validator) Validate(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.ValidationResult, error) {
|
||||
if v == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("validator must not be nil")
|
||||
}
|
||||
if v.llm == nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("LLM client must not be nil")
|
||||
}
|
||||
sourceInput, scene, err := validateRequest(ctx, req)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
|
||||
var response completionResponse
|
||||
_, err = v.llm.CompleteStructured(ctx, contracts.StructuredCompletionRequest{
|
||||
StageName: Key,
|
||||
PromptID: PromptID,
|
||||
PromptVersion: SchemaVersion,
|
||||
ProfileID: req.LLMProfile,
|
||||
SessionID: req.SessionID,
|
||||
StructuredOutputRepairAttempts: req.StructuredOutputRepairAttempts,
|
||||
Inputs: contracts.LLMInputSet{
|
||||
"transcript": shared.TranscriptPromptMaterial(sourceInput),
|
||||
"proposed_kind": contracts.NewLLMInputMaterial("proposed_kind", "text/plain", []byte(scene.Kind), "", ""),
|
||||
},
|
||||
}, &response)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, validatorErrorf("complete structured output: %w", err)
|
||||
}
|
||||
return interpretResponse(response, scene.Kind)
|
||||
}
|
||||
|
||||
func validateRequest(ctx context.Context, req contracts.TypedValidationRequest[dnd.SceneDescriptionList]) (contracts.LLMInputMaterial, dnd.SceneDescription, error) {
|
||||
if ctx == nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context must not be nil")
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("context error before validation: %w", err)
|
||||
}
|
||||
if req.Stage != string(pipeline.StageExtract) {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validation stage must be extract, got %q", req.Stage)
|
||||
}
|
||||
sourceInput, err := shared.PrepareChunkExtraction(ctx, contracts.TypedExtractionRequest{Source: req.Source, Chunk: req.Chunk, SourceInput: req.SourceInput})
|
||||
if err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("prepare chunk source input: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(req.Chunk.ID) == "" || req.Chunk.ID != strings.TrimSpace(req.Chunk.ID) {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk ID must be nonblank and trimmed")
|
||||
}
|
||||
if req.Chunk.SourceID != req.Source.ID {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("current chunk source ID %q does not match source %q", req.Chunk.SourceID, req.Source.ID)
|
||||
}
|
||||
if err := source.NewDocumentIndex(req.Source).ValidateRef(req.Chunk.Ref); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate current chunk source range: %w", err)
|
||||
}
|
||||
if err := shape.ValidateForStage(req.Value, req.Stage); err != nil {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("validate scene description shape: %w", err)
|
||||
}
|
||||
scene := req.Value.Scenes[0]
|
||||
if scene.ID != req.Chunk.ID {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene ID must equal current chunk ID")
|
||||
}
|
||||
if scene.SourceRef != req.Chunk.Ref {
|
||||
return contracts.LLMInputMaterial{}, dnd.SceneDescription{}, validatorErrorf("scene source range must equal current chunk range")
|
||||
}
|
||||
return sourceInput, scene, nil
|
||||
}
|
||||
|
||||
func interpretResponse(response completionResponse, proposedKind dnd.SceneKind) (contracts.ValidationResult, error) {
|
||||
explanation, err := validateExplanation(response.Explanation)
|
||||
if err != nil {
|
||||
return contracts.ValidationResult{}, err
|
||||
}
|
||||
switch response.Verdict {
|
||||
case "approved":
|
||||
return contracts.ValidationResult{Approved: true}, nil
|
||||
case "combat_should_be_added":
|
||||
if proposedKind == dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_added verdict is inconsistent with proposed combat kind")
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeActiveCombatNotClassified,
|
||||
Message: "The current chunk contains substantive active combat that is not classified as combat.",
|
||||
CorrectionGuidance: "Return kind: combat for this scene. " + explanation,
|
||||
}, nil
|
||||
case "combat_should_be_removed":
|
||||
if proposedKind != dnd.SceneKindCombat {
|
||||
return contracts.ValidationResult{}, validatorErrorf("combat_should_be_removed verdict is inconsistent with proposed non-combat kind")
|
||||
}
|
||||
return contracts.ValidationResult{
|
||||
Approved: false,
|
||||
ReasonCode: ReasonCodeCombatClassificationUnsupported,
|
||||
Message: "The current chunk does not support a combat classification.",
|
||||
CorrectionGuidance: "Choose the appropriate narrative, recap, or meta kind for this scene. " + explanation,
|
||||
}, nil
|
||||
default:
|
||||
return contracts.ValidationResult{}, validatorErrorf("unsupported combat-semantics verdict %q", response.Verdict)
|
||||
}
|
||||
}
|
||||
|
||||
func validateExplanation(value string) (string, error) {
|
||||
if !utf8.ValidString(value) {
|
||||
return "", validatorErrorf("validator explanation must be valid UTF-8")
|
||||
}
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", validatorErrorf("validator explanation must not be blank")
|
||||
}
|
||||
if trimmed != value {
|
||||
return "", validatorErrorf("validator explanation must be trimmed")
|
||||
}
|
||||
if utf8.RuneCountInString(value) > maximumExplanationRunes {
|
||||
return "", validatorErrorf("validator explanation exceeds %d Unicode code points", maximumExplanationRunes)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func Spec() pipeline.ValidatorSpec {
|
||||
return pipeline.ValidatorSpec{Key: Key, ExecutionClass: contracts.ExecutionClassLLMBacked}
|
||||
}
|
||||
|
||||
func Register(registry *pipeline.ValidatorRegistry) error {
|
||||
return pipeline.RegisterTypedValidatorBuilder(registry, dnd.SceneDescriptionListKind, Spec(), validateOptions, func(request pipeline.BuildRequest) (contracts.TypedValidator[dnd.SceneDescriptionList], error) {
|
||||
options, err := DecodeOptions(request.Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return New(request.Dependencies.LLM, options)
|
||||
})
|
||||
}
|
||||
|
||||
func DecodeOptions(options map[string]any) (Options, error) {
|
||||
if err := pipeline.RejectUnknownOptions(options); err != nil {
|
||||
return Options{}, err
|
||||
}
|
||||
return Options{}, nil
|
||||
}
|
||||
|
||||
func validateOptions(options map[string]any) error { _, err := DecodeOptions(options); return err }
|
||||
|
||||
func validatorErrorf(format string, args ...any) error {
|
||||
return fmt.Errorf("scene combat-semantics validator: "+format, args...)
|
||||
}
|
||||
Reference in New Issue
Block a user