Execute complete validator chains
This commit is contained in:
207
internal/framework/pipeline/validation_executor_test.go
Normal file
207
internal/framework/pipeline/validation_executor_test.go
Normal file
@@ -0,0 +1,207 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
||||
)
|
||||
|
||||
func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
chain preparedValidatorChain
|
||||
invoke validationInvoker
|
||||
want []validationOutcome
|
||||
wantCalls []string
|
||||
wantGuidance string
|
||||
wantWarnings []string
|
||||
}{
|
||||
{
|
||||
name: "all approved",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{ReasonCode: "shape"}}}}},
|
||||
"refs": {{result: contracts.ValidationResult{Approved: true, Warnings: []contracts.Warning{{ReasonCode: "refs"}}}}},
|
||||
}),
|
||||
want: []validationOutcome{validationApproved, validationApproved},
|
||||
wantCalls: []string{"shape:1", "refs:1"},
|
||||
wantWarnings: []string{"shape", "refs"},
|
||||
},
|
||||
{
|
||||
name: "multiple rejections deduplicate guidance",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}},
|
||||
"refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}},
|
||||
"coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}},
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
|
||||
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
|
||||
wantGuidance: "repair shape\nrepair references",
|
||||
},
|
||||
{
|
||||
name: "rejection and failure both settle",
|
||||
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{
|
||||
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}},
|
||||
"remote": {{err: errors.New("offline")}},
|
||||
}),
|
||||
want: []validationOutcome{validationRejected, validationFailed},
|
||||
wantCalls: []string{"shape:1", "remote:1"},
|
||||
wantGuidance: "repair shape",
|
||||
},
|
||||
{
|
||||
name: "failure only has no correction guidance",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}),
|
||||
want: []validationOutcome{validationFailed},
|
||||
wantCalls: []string{"remote:1"},
|
||||
},
|
||||
{
|
||||
name: "skipped is retained without guidance",
|
||||
chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)),
|
||||
invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}),
|
||||
want: []validationOutcome{validationSkipped},
|
||||
wantCalls: []string{"optional:1"},
|
||||
},
|
||||
{
|
||||
name: "LLM failure retries then succeeds",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}),
|
||||
want: []validationOutcome{validationApproved},
|
||||
wantCalls: []string{"remote:1", "remote:2", "remote:3"},
|
||||
},
|
||||
{
|
||||
name: "LLM failure retry exhaustion records once",
|
||||
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)),
|
||||
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}),
|
||||
want: []validationOutcome{validationFailed},
|
||||
wantCalls: []string{"remote:1", "remote:2"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := []string(nil)
|
||||
invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) {
|
||||
calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt)))
|
||||
return test.invoke(ctx, validator, attempt)
|
||||
}
|
||||
report, err := executeValidationChain(context.Background(), test.chain, invoke)
|
||||
if err != nil {
|
||||
t.Fatalf("executeValidationChain() error = %v", err)
|
||||
}
|
||||
records := report.Records()
|
||||
outcomes := make([]validationOutcome, len(records))
|
||||
for index, record := range records {
|
||||
outcomes[index] = record.outcome
|
||||
}
|
||||
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
|
||||
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
|
||||
}
|
||||
if guidance := report.CorrectionGuidance(); guidance != test.wantGuidance {
|
||||
t.Fatalf("CorrectionGuidance() = %q, want %q", guidance, test.wantGuidance)
|
||||
}
|
||||
warnings := report.Warnings()
|
||||
var warningCodes []string
|
||||
for _, warning := range warnings {
|
||||
warningCodes = append(warningCodes, warning.ReasonCode)
|
||||
}
|
||||
if !reflect.DeepEqual(warningCodes, test.wantWarnings) {
|
||||
t.Fatalf("Warnings() = %#v, want codes %#v", warnings, test.wantWarnings)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
|
||||
candidate := []byte(`{"events":["one"]}`)
|
||||
chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1))
|
||||
var received [][]byte
|
||||
report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) {
|
||||
requestCandidate := append([]byte(nil), candidate...)
|
||||
received = append(received, requestCandidate)
|
||||
requestCandidate[0] = 'x'
|
||||
if attempt == 1 {
|
||||
return validationInvocation{}, errors.New("retry")
|
||||
}
|
||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
||||
})
|
||||
if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 {
|
||||
t.Fatalf("report = %#v, error = %v", report, err)
|
||||
}
|
||||
if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) {
|
||||
t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationReportBoundsGuidanceAndOwnsRecords(t *testing.T) {
|
||||
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
|
||||
report := validationReport{records: make([]validationRecord, 17)}
|
||||
for index := range report.records {
|
||||
unique := []byte(guidance)
|
||||
unique[0] = byte('a' + index)
|
||||
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
|
||||
}
|
||||
report.records[0].warnings = []contracts.Warning{{ReasonCode: "warning"}}
|
||||
if got := report.CorrectionGuidance(); len(got) > contracts.MaxCorrectionGuidanceBytes || !strings.Contains(got, string([]byte{byte('a')})) || strings.Contains(got, string([]byte{byte('q')})) {
|
||||
t.Fatalf("CorrectionGuidance() length = %d, want bounded ordered guidance", len(got))
|
||||
}
|
||||
records := report.Records()
|
||||
records[0].warnings[0].ReasonCode = "changed"
|
||||
if report.Records()[0].warnings[0].ReasonCode != "warning" {
|
||||
t.Fatal("Records() exposed mutable warning storage")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) {
|
||||
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1))
|
||||
framework := errors.New("debug persistence failed")
|
||||
if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
||||
return validationInvocation{}, fatalValidationError(framework)
|
||||
}); !errors.Is(err, framework) {
|
||||
t.Fatalf("framework error = %v, want %v", err, framework)
|
||||
}
|
||||
canceled, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
||||
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
||||
}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("cancellation error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
type validationStep struct {
|
||||
result contracts.ValidationResult
|
||||
invocation validationInvocation
|
||||
err error
|
||||
}
|
||||
|
||||
func validationSequence(steps map[string][]validationStep) validationInvoker {
|
||||
positions := make(map[string]int)
|
||||
return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) {
|
||||
name := validator.resolved.Binding.Module
|
||||
index := positions[name]
|
||||
positions[name]++
|
||||
if index >= len(steps[name]) {
|
||||
return validationInvocation{}, errors.New("unexpected validator invocation")
|
||||
}
|
||||
step := steps[name][index]
|
||||
if step.invocation.skipped {
|
||||
return step.invocation, step.err
|
||||
}
|
||||
return validationInvocation{result: step.result}, step.err
|
||||
}
|
||||
}
|
||||
|
||||
func validationChain(validators ...preparedValidator) preparedValidatorChain {
|
||||
return preparedValidatorChain{validators: validators}
|
||||
}
|
||||
|
||||
func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator {
|
||||
return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}}
|
||||
}
|
||||
Reference in New Issue
Block a user