eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Verify partial validator decisions through CLI workflows

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 11:34:26 +00:00
parent 22f0932389
commit b8089f4a50
6 changed files with 361 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
cmd/audita/main_integration_test.go
View File

@@ -48,18 +48,23 @@ func TestProcessSubprocessHelper(t *testing.T) {
}
type fixtureProvider struct {
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
mode string
secret string
entered chan struct{}
beforeResponse func()
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
validationRequests [][]int
extraDecisionSent bool
mode string
secret string
entered chan struct{}
beforeResponse func()
}
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
var request struct {
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
ResponseFormat struct {
JSONSchema struct {
Schema struct {
@@ -73,6 +78,15 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
return
}
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
var requestedIndices []int
if validation && strings.HasPrefix(p.mode, "decision-") {
var err error
requestedIndices, err = fixtureValidationIndices(request.Messages)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
p.mu.Lock()
p.calls++
if validation {
@@ -82,6 +96,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
validationCall := p.validationCalls
proposalCall := p.proposalCalls
if validation && strings.HasPrefix(p.mode, "decision-") {
p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...))
}
addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent
if addExtra {
p.extraDecisionSent = true
}
beforeResponse := p.beforeResponse
p.mu.Unlock()
if beforeResponse != nil {
@@ -131,6 +152,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
}
}
if strings.HasPrefix(p.mode, "decision-") && !validation {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
if p.mode == "decision-mixed" {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
}
}
if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "repair-filter-later-failure" {
@@ -143,11 +170,53 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
}
}
if validation && strings.HasPrefix(p.mode, "decision-") {
decisions := make([]map[string]any, 0, len(requestedIndices)+2)
for _, index := range requestedIndices {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 {
continue
}
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
}
}
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
}
encoded, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
func fixtureValidationIndices(messages []struct {
Content string `json:"content"`
}) ([]int, error) {
for _, message := range messages {
const marker = "Corrections to validate:"
position := strings.LastIndex(message.Content, marker)
if position < 0 {
continue
}
var items []struct {
CorrectionIndex int `json:"correction_index"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
return nil, fmt.Errorf("decode validation request items: %w", err)
}
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices, nil
}
return nil, fmt.Errorf("validation request items not found")
}
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
type processFixture struct {
@@ -180,11 +249,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
repairs = 1
}
if strings.HasPrefix(mode, "span-") {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
@@ -365,6 +434,240 @@ func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
}
}
func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage {
t.Helper()
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json"))
if err != nil {
t.Fatal(err)
}
var diagnostics []map[string]json.RawMessage
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(raw, &payload); err != nil {
t.Fatal(err)
}
if _, validation := payload["reconciliation"]; validation {
diagnostics = append(diagnostics, payload)
}
}
return diagnostics
}
func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int {
t.Helper()
f.provider.mu.Lock()
requests := append([][]int(nil), f.provider.validationRequests...)
providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls
f.provider.mu.Unlock()
if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations {
t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests)
}
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
if err != nil || len(paths) != 1 {
t.Fatalf("execution path: %v %v", paths, err)
}
raw, err := os.ReadFile(paths[0])
if err != nil {
t.Fatal(err)
}
var execution struct {
Calls []struct {
Summary struct {
ProviderAttempts int `json:"provider_attempts"`
StructuralRepairs int `json:"structural_repairs"`
} `json:"summary"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &execution); err != nil {
t.Fatal(err)
}
if len(execution.Calls) != providerCalls {
t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls)
}
for _, call := range execution.Calls {
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
t.Fatalf("extra generation or structural repair: %+v", call)
}
}
metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json"))
if err != nil {
t.Fatal(err)
}
validatorsSeen := make(map[string]bool)
for _, path := range metadataPaths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var metadata struct {
ValidatorName string `json:"validator_name"`
BatchIndex int `json:"batch_index"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
if metadata.ValidatorName == "" {
continue
}
if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 {
t.Fatalf("duplicate or split validation call: %+v", metadata)
}
validatorsSeen[metadata.ValidatorName] = true
}
if len(validatorsSeen) != validations {
t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations)
}
return requests
}
func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
f := newProcessFixture(t, "decision-mixed", "[grammar]")
f.setSegments(t, "teh", "cat", "bad", "slow")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" {
t.Fatalf("partial validation transcript: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries)
}
if len(ledger) != 4 {
t.Fatalf("unexpected candidate ledger entries: %+v", ledger)
}
for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} {
entry := ledger[index]
var candidate int
var reason string
if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 {
t.Fatal(err)
}
if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) {
t.Fatalf("candidate %d evidence: %+v", index, entry)
}
if index > 0 {
var decisions []struct {
Disposition string `json:"disposition"`
}
if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition {
t.Fatalf("candidate %d decision: %+v %v", index, decisions, err)
}
}
}
var mixedSeen bool
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
UnavailableIndices []int `json:"unavailable_indices"`
IgnoredIndices []int `json:"ignored_indices"`
Decisions []struct {
ProposalIndex int `json:"proposal_index"`
ReasonCode string `json:"reason_code"`
} `json:"decisions"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) {
continue
}
var decoded struct {
Validations []struct {
CorrectionIndex int `json:"correction_index"`
} `json:"validations"`
}
if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil {
t.Fatal(err)
}
var finalContent string
if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" {
t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded)
}
mixedSeen = true
}
if !mixedSeen {
t.Fatal("missing mixed decision interaction evidence")
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
full, survivor := 0, 0
for _, indices := range requests {
switch {
case reflect.DeepEqual(indices, []int{0, 1, 2, 3}):
full++
case reflect.DeepEqual(indices, []int{0}):
survivor++
default:
t.Fatalf("unexpected validation identities: %v", requests)
}
}
if full != 1 || survivor == 0 {
t.Fatalf("wrong validation progression: %v", requests)
}
}
func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
f := newProcessFixture(t, "decision-extra-only", "[grammar]")
f.setSegments(t, "teh")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
if len(output.Segments) != 1 || output.Segments[0].Text != "the" {
t.Fatalf("lost usable correction: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries)
}
markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) {
t.Fatalf("Markdown totals disagree: %s", markdown)
}
_, ledger := f.debugArtifacts(t)
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" {
t.Fatalf("unknown candidate in ledger: %+v", ledger)
}
ignored := 0
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
IgnoredIndices []int `json:"ignored_indices"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) {
ignored++
}
}
if ignored != 1 {
t.Fatalf("expected one handled extra decision, got %d", ignored)
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
for _, indices := range requests {
if !reflect.DeepEqual(indices, []int{0}) {
t.Fatalf("unexpected identities: %v", requests)
}
}
}
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
f := newProcessFixture(t, "span-clean", "[spoken_word]")
f.setSegments(t, "uh hello", "I I think")