From 99391cd18b2341adb885a54382e7d79597e41067 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 23 May 2026 18:02:53 +0000 Subject: [PATCH] Centralize validator classification and malformed output handling --- internal/cli/review_artifacts.go | 24 ++++------ internal/cli/review_artifacts_test.go | 46 +++++++++++++++++++ .../framework/proposal_generation/generate.go | 24 +--------- .../proposal_generation/generate_test.go | 17 +++++++ .../framework/structuredoutput/malformed.go | 28 +++++++++++ .../structuredoutput/malformed_test.go | 32 +++++++++++++ .../framework/validators/llm_validators.go | 24 +--------- .../validators/llm_validators_test.go | 17 +++++++ internal/validators/metadata/metadata.go | 43 ++++++++++++++++- internal/validators/metadata/metadata_test.go | 29 ++++++++++++ internal/validators/registry.go | 30 ++++++------ internal/validators/registry_test.go | 25 ++++++---- 12 files changed, 255 insertions(+), 84 deletions(-) create mode 100644 internal/cli/review_artifacts_test.go create mode 100644 internal/framework/structuredoutput/malformed.go create mode 100644 internal/framework/structuredoutput/malformed_test.go diff --git a/internal/cli/review_artifacts.go b/internal/cli/review_artifacts.go index b8adf83..c216669 100644 --- a/internal/cli/review_artifacts.go +++ b/internal/cli/review_artifacts.go @@ -5,6 +5,7 @@ import ( "sort" "gitea.maximumdirect.net/eric/audita/internal/framework/runner" + validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" ) const ( @@ -48,12 +49,6 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor } entries := make([]correctionLedgerEntry, 0) - llmBacked := map[string]bool{ - "spoken_form_plausibility": true, - "meaning_reversal_review": true, - "editorial_review": true, - } - for _, module := range runOutput.ModuleResults { decisionsByProposal := make(map[int][]runner.ValidatorDecisionRecord) for _, decision := range module.ValidatorDecisions { @@ -72,8 +67,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor AppliedCorrectedText: change.CorrectedText, ReplacementPolicy: string(module.ReplacementPolicy), Disposition: correctionDispositionApplied, - DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked), - LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked), + DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false), + LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true), }) } for _, change := range module.SkippedChanges { @@ -89,8 +84,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor Disposition: correctionDispositionSkipped, DispositionReasonCode: string(change.SkipReason), DispositionMessage: change.Message, - DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false, llmBacked), - LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true, llmBacked), + DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], false), + LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[change.ProposalIndex], true), }) } for _, rejection := range module.ValidatorRejected { @@ -106,8 +101,8 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor Disposition: correctionDispositionRejected, DispositionReasonCode: rejection.ReasonCode, DispositionMessage: rejection.Message, - DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false, llmBacked), - LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true, llmBacked), + DeterministicValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], false), + LLMValidatorResults: filterLedgerDecisions(decisionsByProposal[rejection.ProposalIndex], true), }) } if module.Status == runner.ModuleStatusFailed { @@ -135,13 +130,14 @@ func buildCorrectionLedger(runDirPath string, runOutput *runner.RunOutput) []cor return entries } -func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool, llmBacked map[string]bool) []ledgerValidatorDecisionRecord { +func filterLedgerDecisions(in []runner.ValidatorDecisionRecord, wantLLM bool) []ledgerValidatorDecisionRecord { if len(in) == 0 { return nil } out := make([]ledgerValidatorDecisionRecord, 0, len(in)) for _, decision := range in { - if llmBacked[decision.ValidatorName] != wantLLM { + isLLMBacked := validatormetadata.ClassForKey(decision.ValidatorName) == validatormetadata.ExecutionClassLLMBacked + if isLLMBacked != wantLLM { continue } out = append(out, ledgerValidatorDecisionRecord{ diff --git a/internal/cli/review_artifacts_test.go b/internal/cli/review_artifacts_test.go new file mode 100644 index 0000000..0523392 --- /dev/null +++ b/internal/cli/review_artifacts_test.go @@ -0,0 +1,46 @@ +package cli + +import ( + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" + "gitea.maximumdirect.net/eric/audita/internal/framework/runner" +) + +func TestBuildCorrectionLedgerClassifiesValidatorDecisionsFromCanonicalMetadata(t *testing.T) { + output := &runner.RunOutput{ + ModuleResults: []runner.ModuleResult{ + { + ModuleKey: "glossary", + ModuleInstance: "glossary", + ReplacementPolicy: proposals.ReplacementPolicyReplaceAll, + ValidatorDecisions: []runner.ValidatorDecisionRecord{ + {ValidatorName: "proposal_shape", ProposalIndex: 3, Approved: true, ReasonCode: "approved"}, + {ValidatorName: "spoken_form_plausibility", ProposalIndex: 3, Approved: true, ReasonCode: "approved"}, + }, + AppliedChanges: []proposals.AppliedChange{ + { + ProposalIndex: 3, + ModuleKey: "glossary", + ModuleInstance: "glossary", + TargetSegmentID: 1, + OriginalText: "gestures", + CorrectedText: "Jesters", + }, + }, + }, + }, + } + + ledger := buildCorrectionLedger("/tmp/audita-run-id", output) + if len(ledger) != 1 { + t.Fatalf("expected one ledger entry, got %d", len(ledger)) + } + entry := ledger[0] + if len(entry.DeterministicValidatorResults) != 1 || entry.DeterministicValidatorResults[0].ValidatorKey != "proposal_shape" { + t.Fatalf("unexpected deterministic decision split: %+v", entry.DeterministicValidatorResults) + } + if len(entry.LLMValidatorResults) != 1 || entry.LLMValidatorResults[0].ValidatorKey != "spoken_form_plausibility" { + t.Fatalf("unexpected llm-backed decision split: %+v", entry.LLMValidatorResults) + } +} diff --git a/internal/framework/proposal_generation/generate.go b/internal/framework/proposal_generation/generate.go index bcba659..56e2bdf 100644 --- a/internal/framework/proposal_generation/generate.go +++ b/internal/framework/proposal_generation/generate.go @@ -16,6 +16,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" ) @@ -159,7 +160,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) { } if callErr != nil { - if isMalformedStructuredOutputError(callErr) { + if structuredoutput.IsMalformedError(callErr) { return Result{ Warnings: []stagewarnings.StageWarning{newMalformedProposalWarning(req.Section, artifacts, callErr)}, Artifacts: artifacts, @@ -230,27 +231,6 @@ func errPayload(err error) any { return map[string]any{"error": err.Error()} } -func isMalformedStructuredOutputError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - for _, marker := range []string{ - "malformed structured output", - "decode structured output:", - "decode provider response envelope:", - "provider response missing choices", - "provider response missing assistant message content", - "provider response assistant message content is empty", - "provider response assistant message content is not valid JSON", - } { - if strings.Contains(msg, marker) { - return true - } - } - return false -} - func newMalformedProposalWarning(section *contracts.SectionMetadata, artifacts InteractionArtifacts, err error) stagewarnings.StageWarning { warning := stagewarnings.StageWarning{ Scope: stagewarnings.ScopeProposalGeneration, diff --git a/internal/framework/proposal_generation/generate_test.go b/internal/framework/proposal_generation/generate_test.go index 0cc53ce..d2c734c 100644 --- a/internal/framework/proposal_generation/generate_test.go +++ b/internal/framework/proposal_generation/generate_test.go @@ -272,6 +272,23 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) } } +func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) { + client := &fakeStructuredClient{err: errors.New("provider response missing choices")} + req := defaultRequest(t) + req.LLMClient = client + + result, err := GenerateCandidates(context.Background(), req) + if err != nil { + t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err) + } + if len(result.Corrections) != 0 || len(result.Enriched) != 0 { + t.Fatalf("expected no proposals on malformed response, got %+v", result) + } + if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" { + t.Fatalf("unexpected warnings: %+v", result.Warnings) + } +} + func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) { baseResponse := StructuredCorrectionSet{ Corrections: []StructuredCorrectionProposal{ diff --git a/internal/framework/structuredoutput/malformed.go b/internal/framework/structuredoutput/malformed.go new file mode 100644 index 0000000..01fe815 --- /dev/null +++ b/internal/framework/structuredoutput/malformed.go @@ -0,0 +1,28 @@ +package structuredoutput + +import "strings" + +var malformedMarkers = []string{ + "malformed structured output", + "decode structured output:", + "decode provider response envelope:", + "provider response missing choices", + "provider response missing assistant message content", + "provider response assistant message content is empty", + "provider response assistant message content is not valid JSON", +} + +// IsMalformedError reports whether err matches provider malformed +// structured-output failure markers that should be downgraded. +func IsMalformedError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + for _, marker := range malformedMarkers { + if strings.Contains(msg, marker) { + return true + } + } + return false +} diff --git a/internal/framework/structuredoutput/malformed_test.go b/internal/framework/structuredoutput/malformed_test.go new file mode 100644 index 0000000..fd5c08a --- /dev/null +++ b/internal/framework/structuredoutput/malformed_test.go @@ -0,0 +1,32 @@ +package structuredoutput + +import ( + "errors" + "testing" +) + +func TestIsMalformedError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {name: "nil", err: nil, want: false}, + {name: "generic", err: errors.New("network timeout"), want: false}, + {name: "malformed", err: errors.New("malformed structured output"), want: true}, + {name: "decode structured", err: errors.New("decode structured output: unexpected end of JSON input"), want: true}, + {name: "missing choices", err: errors.New("provider response missing choices"), want: true}, + {name: "missing content", err: errors.New("provider response missing assistant message content"), want: true}, + {name: "empty content", err: errors.New("provider response assistant message content is empty"), want: true}, + {name: "invalid content json", err: errors.New("provider response assistant message content is not valid JSON"), want: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := IsMalformedError(tc.err) + if got != tc.want { + t.Fatalf("IsMalformedError(%v): got=%v want=%v", tc.err, got, tc.want) + } + }) + } +} diff --git a/internal/framework/validators/llm_validators.go b/internal/framework/validators/llm_validators.go index eca9343..c12d4de 100644 --- a/internal/framework/validators/llm_validators.go +++ b/internal/framework/validators/llm_validators.go @@ -12,6 +12,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/framework/proposals" "gitea.maximumdirect.net/eric/audita/internal/framework/responseschema" "gitea.maximumdirect.net/eric/audita/internal/framework/stagename" + "gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput" stagewarnings "gitea.maximumdirect.net/eric/audita/internal/framework/warnings" "gitea.maximumdirect.net/eric/audita/internal/prompts" ) @@ -144,7 +145,7 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result, ) } if err != nil { - if isMalformedStructuredOutputError(err) { + if structuredoutput.IsMalformedError(err) { llmDecisions = append(llmDecisions, rejectBatch(batch.Items, ReasonValidatorMalformed, fmt.Sprintf("validator response malformed: %s", strings.TrimSpace(err.Error())))...) warnings = append(warnings, newValidatorWarning(v.name, batch.BatchIndex, ReasonValidatorMalformed, err.Error(), artifacts)) continue @@ -387,24 +388,3 @@ func diagnosticArtifactPath(artifacts InteractionArtifacts) string { } return artifacts.ResponsePayloadPath } - -func isMalformedStructuredOutputError(err error) bool { - if err == nil { - return false - } - msg := err.Error() - for _, marker := range []string{ - "malformed structured output", - "decode structured output:", - "decode provider response envelope:", - "provider response missing choices", - "provider response missing assistant message content", - "provider response assistant message content is empty", - "provider response assistant message content is not valid JSON", - } { - if strings.Contains(msg, marker) { - return true - } - } - return false -} diff --git a/internal/framework/validators/llm_validators_test.go b/internal/framework/validators/llm_validators_test.go index df98712..3c7b449 100644 --- a/internal/framework/validators/llm_validators_test.go +++ b/internal/framework/validators/llm_validators_test.go @@ -265,6 +265,23 @@ func TestLLMBackedValidatorMalformedOutputRejectsBatch(t *testing.T) { } } +func TestLLMBackedValidatorProviderMalformedEnvelopeRejectsBatch(t *testing.T) { + client := &fakeStructuredLLMClient{err: errors.New("provider response assistant message content is empty")} + v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") + req := makeReq([]proposals.EnrichedCorrectionProposal{mk(0, "gestures", "Jesters")}) + req.LLMClient = client + res, err := v.Validate(context.Background(), req) + if err != nil { + t.Fatalf("expected malformed provider envelope downgrade, got %v", err) + } + if len(res.Decisions) != 1 || res.Decisions[0].Approved || res.Decisions[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("unexpected decisions: %+v", res.Decisions) + } + if len(res.Warnings) != 1 || res.Warnings[0].ReasonCode != ReasonValidatorMalformed { + t.Fatalf("expected malformed warning, got %+v", res.Warnings) + } +} + func TestLLMBackedValidatorMissingDecisionRejectsBatch(t *testing.T) { client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{}}}} v, _ := NewLLMBackedValidator("spoken_form_plausibility_review", LLMValidatorTypeSpokenFormPlausibility, "test-model") diff --git a/internal/validators/metadata/metadata.go b/internal/validators/metadata/metadata.go index 681db3a..aa0a745 100644 --- a/internal/validators/metadata/metadata.go +++ b/internal/validators/metadata/metadata.go @@ -1,6 +1,10 @@ package metadata -import "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" +import ( + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/framework/contracts" +) type ExecutionClass string @@ -9,6 +13,31 @@ const ( ExecutionClassLLMBacked ExecutionClass = "llm_backed" ) +const ( + KeyProposalShape = "proposal_shape" + KeyConfidenceThreshold = "confidence_threshold" + KeyOriginalTextPresence = "original_text_presence" + KeyNonEmptyCorrectedText = "non_empty_corrected_text" + KeyNoEffect = "no_effect" + KeyProtectedTerms = "protected_terms" + + KeySpokenFormPlausibility = "spoken_form_plausibility" + KeyMeaningReversalReview = "meaning_reversal_review" + KeyEditorialReview = "editorial_review" +) + +var executionClassByKey = map[string]ExecutionClass{ + KeyProposalShape: ExecutionClassDeterministic, + KeyConfidenceThreshold: ExecutionClassDeterministic, + KeyOriginalTextPresence: ExecutionClassDeterministic, + KeyNonEmptyCorrectedText: ExecutionClassDeterministic, + KeyNoEffect: ExecutionClassDeterministic, + KeyProtectedTerms: ExecutionClassDeterministic, + KeySpokenFormPlausibility: ExecutionClassLLMBacked, + KeyMeaningReversalReview: ExecutionClassLLMBacked, + KeyEditorialReview: ExecutionClassLLMBacked, +} + type ClassifiedValidator interface { contracts.Validator ExecutionClass() ExecutionClass @@ -18,9 +47,11 @@ func ClassOf(v contracts.Validator) ExecutionClass { if v == nil { return ExecutionClassDeterministic } + + classFromKey := ClassForKey(v.Name()) classified, ok := v.(ClassifiedValidator) if !ok { - return ExecutionClassDeterministic + return classFromKey } switch classified.ExecutionClass() { case ExecutionClassLLMBacked: @@ -28,8 +59,16 @@ func ClassOf(v contracts.Validator) ExecutionClass { case ExecutionClassDeterministic: return ExecutionClassDeterministic default: + return classFromKey + } +} + +func ClassForKey(key string) ExecutionClass { + class, ok := executionClassByKey[strings.TrimSpace(key)] + if !ok { return ExecutionClassDeterministic } + return class } func Wrap(v contracts.Validator, class ExecutionClass) contracts.Validator { diff --git a/internal/validators/metadata/metadata_test.go b/internal/validators/metadata/metadata_test.go index d19870a..42eb57b 100644 --- a/internal/validators/metadata/metadata_test.go +++ b/internal/validators/metadata/metadata_test.go @@ -16,6 +16,16 @@ func (u unclassifiedValidator) Validate(_ context.Context, _ contracts.Validatio return frameworkvalidators.Result{ValidatorName: u.Name(), Decisions: nil}, nil } +type namedUnclassifiedValidator struct { + name string +} + +func (n namedUnclassifiedValidator) Name() string { return n.name } + +func (n namedUnclassifiedValidator) Validate(_ context.Context, _ contracts.ValidationRequest) (frameworkvalidators.Result, error) { + return frameworkvalidators.Result{ValidatorName: n.Name(), Decisions: nil}, nil +} + func TestClassOfDefaultsToDeterministic(t *testing.T) { if got := ClassOf(unclassifiedValidator{}); got != ExecutionClassDeterministic { t.Fatalf("expected deterministic default class, got %q", got) @@ -28,3 +38,22 @@ func TestWrapExposesExecutionClass(t *testing.T) { t.Fatalf("expected llm_backed class, got %q", got) } } + +func TestClassForKey(t *testing.T) { + if got := ClassForKey(KeyProposalShape); got != ExecutionClassDeterministic { + t.Fatalf("expected deterministic class for %q, got %q", KeyProposalShape, got) + } + if got := ClassForKey(KeySpokenFormPlausibility); got != ExecutionClassLLMBacked { + t.Fatalf("expected llm_backed class for %q, got %q", KeySpokenFormPlausibility, got) + } + if got := ClassForKey("unknown"); got != ExecutionClassDeterministic { + t.Fatalf("expected deterministic fallback for unknown key, got %q", got) + } +} + +func TestClassOfFallsBackToStableValidatorKey(t *testing.T) { + v := namedUnclassifiedValidator{name: KeyEditorialReview} + if got := ClassOf(v); got != ExecutionClassLLMBacked { + t.Fatalf("expected llm_backed fallback by key, got %q", got) + } +} diff --git a/internal/validators/registry.go b/internal/validators/registry.go index 4763319..fda55f0 100644 --- a/internal/validators/registry.go +++ b/internal/validators/registry.go @@ -8,6 +8,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/validators/confidence_threshold" "gitea.maximumdirect.net/eric/audita/internal/validators/editorial_review" "gitea.maximumdirect.net/eric/audita/internal/validators/meaning_reversal_review" + validatormetadata "gitea.maximumdirect.net/eric/audita/internal/validators/metadata" "gitea.maximumdirect.net/eric/audita/internal/validators/no_effect" "gitea.maximumdirect.net/eric/audita/internal/validators/non_empty_corrected_text" "gitea.maximumdirect.net/eric/audita/internal/validators/original_text_presence" @@ -17,22 +18,21 @@ import ( ) const ( - KeyProposalShape = "proposal_shape" - KeyConfidenceThreshold = "confidence_threshold" - KeyOriginalTextPresence = "original_text_presence" - KeyNonEmptyCorrectedText = "non_empty_corrected_text" - KeyNoEffect = "no_effect" - KeyProtectedTerms = "protected_terms" + KeyProposalShape = validatormetadata.KeyProposalShape + KeyConfidenceThreshold = validatormetadata.KeyConfidenceThreshold + KeyOriginalTextPresence = validatormetadata.KeyOriginalTextPresence + KeyNonEmptyCorrectedText = validatormetadata.KeyNonEmptyCorrectedText + KeyNoEffect = validatormetadata.KeyNoEffect + KeyProtectedTerms = validatormetadata.KeyProtectedTerms - KeySpokenFormPlausibility = "spoken_form_plausibility" - KeyMeaningReversalReview = "meaning_reversal_review" - KeyEditorialReview = "editorial_review" + KeySpokenFormPlausibility = validatormetadata.KeySpokenFormPlausibility + KeyMeaningReversalReview = validatormetadata.KeyMeaningReversalReview + KeyEditorialReview = validatormetadata.KeyEditorialReview ) type BuiltInValidatorDefinition struct { - Key string - Build func() (contracts.Validator, error) - LLMBacked bool + Key string + Build func() (contracts.Validator, error) } type Registry struct { @@ -47,9 +47,9 @@ func NewBuiltInRegistry() *Registry { {Key: KeyNonEmptyCorrectedText, Build: non_empty_corrected_text.New}, {Key: KeyNoEffect, Build: no_effect.New}, {Key: KeyProtectedTerms, Build: protected_terms.New}, - {Key: KeySpokenFormPlausibility, LLMBacked: true, Build: spoken_form_plausibility.New}, - {Key: KeyMeaningReversalReview, LLMBacked: true, Build: meaning_reversal_review.New}, - {Key: KeyEditorialReview, LLMBacked: true, Build: editorial_review.New}, + {Key: KeySpokenFormPlausibility, Build: spoken_form_plausibility.New}, + {Key: KeyMeaningReversalReview, Build: meaning_reversal_review.New}, + {Key: KeyEditorialReview, Build: editorial_review.New}, } m := make(map[string]BuiltInValidatorDefinition, len(defs)) diff --git a/internal/validators/registry_test.go b/internal/validators/registry_test.go index 285a70f..85ff5ab 100644 --- a/internal/validators/registry_test.go +++ b/internal/validators/registry_test.go @@ -82,11 +82,6 @@ func TestBuiltInValidatorPackagesConstruct(t *testing.T) { func TestRegistryBuildsClassifiedValidators(t *testing.T) { r := NewBuiltInRegistry() - llmKeys := map[string]bool{ - KeySpokenFormPlausibility: true, - KeyMeaningReversalReview: true, - KeyEditorialReview: true, - } for _, key := range r.RegisteredKeys() { v, err := r.MustBuild(key) if err != nil { @@ -95,16 +90,28 @@ func TestRegistryBuildsClassifiedValidators(t *testing.T) { if _, ok := v.(validatormetadata.ClassifiedValidator); !ok { t.Fatalf("expected built validator %q to expose execution classification metadata", key) } - want := validatormetadata.ExecutionClassDeterministic - if llmKeys[key] { - want = validatormetadata.ExecutionClassLLMBacked - } + want := validatormetadata.ClassForKey(key) if got := validatormetadata.ClassOf(v); got != want { t.Fatalf("expected class %q for %q, got %q", want, key, got) } } } +func TestExecutionClassResolvableByStableKeyAndByValidatorInstance(t *testing.T) { + r := NewBuiltInRegistry() + for _, key := range r.RegisteredKeys() { + v, err := r.MustBuild(key) + if err != nil { + t.Fatalf("must build %q: %v", key, err) + } + fromKey := validatormetadata.ClassForKey(key) + fromInstance := validatormetadata.ClassOf(v) + if fromInstance != fromKey { + t.Fatalf("class mismatch for %q: key=%q instance=%q", key, fromKey, fromInstance) + } + } +} + func TestRegistryProtectedTermsUsesNonGlossaryStageBehavior(t *testing.T) { r := NewBuiltInRegistry() v, err := r.MustBuild(KeyProtectedTerms)