242 lines
11 KiB
Go
242 lines
11 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func TestExecuteValidationChainSettlesEveryValidatorInOrder(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
chain preparedValidatorChain
|
|
invoke validationInvoker
|
|
want []validationOutcome
|
|
wantCalls []string
|
|
wantGuidance []string
|
|
wantDiagnostics []string
|
|
}{
|
|
{
|
|
name: "all approved",
|
|
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0)),
|
|
invoke: validationSequence(map[string][]validationStep{
|
|
"shape": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("shape")}}}},
|
|
"refs": {{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{validationDiagnostic("refs")}}}},
|
|
}),
|
|
want: []validationOutcome{validationApproved, validationApproved},
|
|
wantCalls: []string{"shape:1", "refs:1"},
|
|
wantDiagnostics: []string{"shape", "refs"},
|
|
},
|
|
{
|
|
name: "multiple rejections deduplicate guidance",
|
|
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("refs", contracts.ExecutionClassDeterministic, 0), validationSpec("coverage", contracts.ExecutionClassDeterministic, 0)),
|
|
invoke: validationSequence(map[string][]validationStep{
|
|
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", Message: "invalid", CorrectionGuidance: "repair shape"}}},
|
|
"refs": {{result: contracts.ValidationResult{ReasonCode: "refs", Message: "missing", CorrectionGuidance: "repair references"}}},
|
|
"coverage": {{result: contracts.ValidationResult{ReasonCode: "coverage", Message: "missing", CorrectionGuidance: "repair shape"}}},
|
|
}),
|
|
want: []validationOutcome{validationRejected, validationRejected, validationRejected},
|
|
wantCalls: []string{"shape:1", "refs:1", "coverage:1"},
|
|
wantGuidance: []string{"repair shape", "repair references"},
|
|
},
|
|
{
|
|
name: "rejection and failure both settle",
|
|
chain: validationChain(validationSpec("shape", contracts.ExecutionClassDeterministic, 0), validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
|
invoke: validationSequence(map[string][]validationStep{
|
|
"shape": {{result: contracts.ValidationResult{ReasonCode: "shape", CorrectionGuidance: "repair shape"}}},
|
|
"remote": {{err: errors.New("offline")}},
|
|
}),
|
|
want: []validationOutcome{validationRejected, validationFailed},
|
|
wantCalls: []string{"shape:1", "remote:1"},
|
|
wantGuidance: []string{"repair shape"},
|
|
},
|
|
{
|
|
name: "failure only has no correction guidance",
|
|
chain: validationChain(validationSpec("remote", contracts.ExecutionClassDeterministic, 0)),
|
|
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("offline")}}}),
|
|
want: []validationOutcome{validationFailed},
|
|
wantCalls: []string{"remote:1"},
|
|
},
|
|
{
|
|
name: "skipped is retained without guidance",
|
|
chain: validationChain(validationSpec("optional", contracts.ExecutionClassDeterministic, 0)),
|
|
invoke: validationSequence(map[string][]validationStep{"optional": {{invocation: skippedValidation("runtime_unavailable", "optional dependency unavailable")}}}),
|
|
want: []validationOutcome{validationSkipped},
|
|
wantCalls: []string{"optional:1"},
|
|
},
|
|
{
|
|
name: "LLM failure retries then succeeds",
|
|
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 2)),
|
|
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}, {result: contracts.ValidationResult{Approved: true}}}}),
|
|
want: []validationOutcome{validationApproved},
|
|
wantCalls: []string{"remote:1", "remote:2", "remote:3"},
|
|
},
|
|
{
|
|
name: "LLM failure retry exhaustion records once",
|
|
chain: validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1)),
|
|
invoke: validationSequence(map[string][]validationStep{"remote": {{err: errors.New("first")}, {err: errors.New("second")}}}),
|
|
want: []validationOutcome{validationFailed},
|
|
wantCalls: []string{"remote:1", "remote:2"},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
calls := []string(nil)
|
|
invoke := func(ctx context.Context, validator preparedValidator, attempt int) (validationInvocation, error) {
|
|
calls = append(calls, validator.resolved.Binding.Module+":"+string(rune('0'+attempt)))
|
|
return test.invoke(ctx, validator, attempt)
|
|
}
|
|
report, err := executeValidationChain(context.Background(), test.chain, invoke)
|
|
if err != nil {
|
|
t.Fatalf("executeValidationChain() error = %v", err)
|
|
}
|
|
records := report.Records()
|
|
outcomes := make([]validationOutcome, len(records))
|
|
for index, record := range records {
|
|
outcomes[index] = record.outcome
|
|
}
|
|
if !reflect.DeepEqual(outcomes, test.want) || !reflect.DeepEqual(calls, test.wantCalls) {
|
|
t.Fatalf("outcomes = %#v calls = %#v, want %#v %#v", outcomes, calls, test.want, test.wantCalls)
|
|
}
|
|
if len(test.wantGuidance) > 0 {
|
|
guidance, err := report.CorrectionRequest()
|
|
if err != nil {
|
|
t.Fatalf("CorrectionRequest() error = %v", err)
|
|
}
|
|
if !strings.Contains(guidance, "complete corrected replacement") {
|
|
t.Fatalf("CorrectionRequest() = %q, want complete replacement instruction", guidance)
|
|
}
|
|
for _, item := range test.wantGuidance {
|
|
if strings.Count(guidance, item) != 1 {
|
|
t.Fatalf("CorrectionRequest() = %q, want one occurrence of %q", guidance, item)
|
|
}
|
|
}
|
|
}
|
|
diagnostics := report.Diagnostics()
|
|
var diagnosticCodes []string
|
|
for _, record := range diagnostics {
|
|
diagnosticCodes = append(diagnosticCodes, record.diagnostic.ReasonCode)
|
|
}
|
|
if !reflect.DeepEqual(diagnosticCodes, test.wantDiagnostics) {
|
|
t.Fatalf("Diagnostics() = %#v, want codes %#v", diagnostics, test.wantDiagnostics)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExecuteValidationChainKeepsInvocationInputsImmutable(t *testing.T) {
|
|
candidate := []byte(`{"events":["one"]}`)
|
|
chain := validationChain(validationSpec("remote", contracts.ExecutionClassLLMBacked, 1))
|
|
var received [][]byte
|
|
report, err := executeValidationChain(context.Background(), chain, func(_ context.Context, _ preparedValidator, attempt int) (validationInvocation, error) {
|
|
requestCandidate := append([]byte(nil), candidate...)
|
|
received = append(received, requestCandidate)
|
|
requestCandidate[0] = 'x'
|
|
if attempt == 1 {
|
|
return validationInvocation{}, errors.New("retry")
|
|
}
|
|
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
|
})
|
|
if err != nil || len(report.Records()) != 1 || report.Records()[0].attemptCount != 2 {
|
|
t.Fatalf("report = %#v, error = %v", report, err)
|
|
}
|
|
if string(candidate) != `{"events":["one"]}` || len(received) != 2 || string(received[0]) != string(received[1]) {
|
|
t.Fatalf("candidate = %q received = %#v, want immutable repeated request content", candidate, received)
|
|
}
|
|
}
|
|
|
|
func TestValidationReportRejectsOversizedGuidanceAndOwnsRecords(t *testing.T) {
|
|
guidance := strings.Repeat("x", contracts.MaxValidationCorrectionGuidanceBytes)
|
|
report := validationReport{records: make([]validationRecord, 17)}
|
|
for index := range report.records {
|
|
unique := []byte(guidance)
|
|
unique[0] = byte('a' + index)
|
|
report.records[index] = validationRecord{validatorName: "validator", outcome: validationRejected, correctionGuidance: string(unique)}
|
|
}
|
|
report.records[0].diagnostics = []contracts.ProducerDiagnostic{validationDiagnostic("diagnostic")}
|
|
if _, err := report.CorrectionRequest(); err == nil {
|
|
t.Fatal("CorrectionRequest() error = nil, want aggregate overflow error")
|
|
}
|
|
records := report.Records()
|
|
records[0].diagnostics[0].ReasonCode = "changed"
|
|
if report.Records()[0].diagnostics[0].ReasonCode != "diagnostic" {
|
|
t.Fatal("Records() exposed mutable diagnostic storage")
|
|
}
|
|
}
|
|
|
|
func validationDiagnostic(reasonCode string) contracts.ProducerDiagnostic {
|
|
return contracts.ProducerDiagnostic{
|
|
Disposition: contracts.DiagnosticDispositionObservation,
|
|
Category: contracts.DiagnosticCategoryNormalization,
|
|
ReasonCode: reasonCode,
|
|
OccurrenceCount: 1,
|
|
Samples: []contracts.DiagnosticSample{{Scope: "validator", Message: reasonCode}},
|
|
}
|
|
}
|
|
|
|
func TestExecuteValidationChainReturnsFrameworkAndCancellationErrors(t *testing.T) {
|
|
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 1))
|
|
framework := errors.New("debug persistence failed")
|
|
if _, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
return validationInvocation{}, fatalValidationError(framework)
|
|
}); !errors.Is(err, framework) {
|
|
t.Fatalf("framework error = %v, want %v", err, framework)
|
|
}
|
|
canceled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if _, err := executeValidationChain(canceled, chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
return validationInvocation{result: contracts.ValidationResult{Approved: true}}, nil
|
|
}); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("cancellation error = %v, want context canceled", err)
|
|
}
|
|
}
|
|
|
|
func TestExecuteValidationChainRejectsInvalidDiagnosticsWithoutRetry(t *testing.T) {
|
|
chain := validationChain(validationSpec("validator", contracts.ExecutionClassLLMBacked, 2))
|
|
calls := 0
|
|
_, err := executeValidationChain(context.Background(), chain, func(context.Context, preparedValidator, int) (validationInvocation, error) {
|
|
calls++
|
|
return validationInvocation{result: contracts.ValidationResult{Approved: true, Diagnostics: []contracts.ProducerDiagnostic{{}}}}, nil
|
|
})
|
|
var frameworkErr validationFrameworkError
|
|
if !errors.As(err, &frameworkErr) || calls != 1 || !strings.Contains(err.Error(), "validator \"validator\" returned invalid diagnostics") {
|
|
t.Fatalf("error = %v calls = %d, want one fatal invalid-diagnostics result", err, calls)
|
|
}
|
|
}
|
|
|
|
type validationStep struct {
|
|
result contracts.ValidationResult
|
|
invocation validationInvocation
|
|
err error
|
|
}
|
|
|
|
func validationSequence(steps map[string][]validationStep) validationInvoker {
|
|
positions := make(map[string]int)
|
|
return func(_ context.Context, validator preparedValidator, _ int) (validationInvocation, error) {
|
|
name := validator.resolved.Binding.Module
|
|
index := positions[name]
|
|
positions[name]++
|
|
if index >= len(steps[name]) {
|
|
return validationInvocation{}, errors.New("unexpected validator invocation")
|
|
}
|
|
step := steps[name][index]
|
|
if step.invocation.skipped {
|
|
return step.invocation, step.err
|
|
}
|
|
return validationInvocation{result: step.result}, step.err
|
|
}
|
|
}
|
|
|
|
func validationChain(validators ...preparedValidator) preparedValidatorChain {
|
|
return preparedValidatorChain{validators: validators}
|
|
}
|
|
|
|
func validationSpec(name string, class contracts.ExecutionClass, retries int) preparedValidator {
|
|
return preparedValidator{resolved: ResolvedValidator{Binding: ModuleBinding{Module: name, Retries: retries}, ExecutionClass: class}}
|
|
}
|