diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index ce4879b..c40aa3b 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -48,18 +48,23 @@ func TestProcessSubprocessHelper(t *testing.T) { } type fixtureProvider struct { - mu sync.Mutex - calls int - proposalCalls int - validationCalls int - mode string - secret string - entered chan struct{} - beforeResponse func() + mu sync.Mutex + calls int + proposalCalls int + validationCalls int + validationRequests [][]int + extraDecisionSent bool + mode string + secret string + entered chan struct{} + beforeResponse func() } func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { var request struct { + Messages []struct { + Content string `json:"content"` + } `json:"messages"` ResponseFormat struct { JSONSchema struct { Schema struct { @@ -73,6 +78,15 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { return } _, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"] + var requestedIndices []int + if validation && strings.HasPrefix(p.mode, "decision-") { + var err error + requestedIndices, err = fixtureValidationIndices(request.Messages) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } p.mu.Lock() p.calls++ if validation { @@ -82,6 +96,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } validationCall := p.validationCalls proposalCall := p.proposalCalls + if validation && strings.HasPrefix(p.mode, "decision-") { + p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...)) + } + addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent + if addExtra { + p.extraDecisionSent = true + } beforeResponse := p.beforeResponse p.mu.Unlock() if beforeResponse != nil { @@ -131,6 +152,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } } } + if strings.HasPrefix(p.mode, "decision-") && !validation { + content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}` + if p.mode == "decision-mixed" { + content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}` + } + } if validation && !strings.HasPrefix(p.mode, "span-") { content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` if p.mode == "repair-filter-later-failure" { @@ -143,11 +170,53 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { } } } + if validation && strings.HasPrefix(p.mode, "decision-") { + decisions := make([]map[string]any, 0, len(requestedIndices)+2) + for _, index := range requestedIndices { + if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 { + continue + } + approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1) + decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"}) + if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 { + decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"}) + } + } + if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra { + decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"}) + } + raw, _ := json.Marshal(map[string]any{"validations": decisions}) + content = string(raw) + } encoded, _ := json.Marshal(content) w.Header().Set("Content-Type", "application/json") _, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded) } +func fixtureValidationIndices(messages []struct { + Content string `json:"content"` +}) ([]int, error) { + for _, message := range messages { + const marker = "Corrections to validate:" + position := strings.LastIndex(message.Content, marker) + if position < 0 { + continue + } + var items []struct { + CorrectionIndex int `json:"correction_index"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil { + return nil, fmt.Errorf("decode validation request items: %w", err) + } + indices := make([]int, len(items)) + for i, item := range items { + indices[i] = item.CorrectionIndex + } + return indices, nil + } + return nil, fmt.Errorf("validation request items not found") +} + func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls } type processFixture struct { @@ -180,11 +249,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" { repairs = 1 } - if strings.HasPrefix(mode, "span-") { + if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { repairs = 1 } configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs")) - if strings.HasPrefix(mode, "span-") { + if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") { configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" } if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil { @@ -365,6 +434,240 @@ func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) { } } +func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage { + t.Helper() + paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json")) + if err != nil { + t.Fatal(err) + } + var diagnostics []map[string]json.RawMessage + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var payload map[string]json.RawMessage + if err := json.Unmarshal(raw, &payload); err != nil { + t.Fatal(err) + } + if _, validation := payload["reconciliation"]; validation { + diagnostics = append(diagnostics, payload) + } + } + return diagnostics +} + +func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int { + t.Helper() + f.provider.mu.Lock() + requests := append([][]int(nil), f.provider.validationRequests...) + providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls + f.provider.mu.Unlock() + if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations { + t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests) + } + paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json")) + if err != nil || len(paths) != 1 { + t.Fatalf("execution path: %v %v", paths, err) + } + raw, err := os.ReadFile(paths[0]) + if err != nil { + t.Fatal(err) + } + var execution struct { + Calls []struct { + Summary struct { + ProviderAttempts int `json:"provider_attempts"` + StructuralRepairs int `json:"structural_repairs"` + } `json:"summary"` + } `json:"calls"` + } + if err := json.Unmarshal(raw, &execution); err != nil { + t.Fatal(err) + } + if len(execution.Calls) != providerCalls { + t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls) + } + for _, call := range execution.Calls { + if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 { + t.Fatalf("extra generation or structural repair: %+v", call) + } + } + metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json")) + if err != nil { + t.Fatal(err) + } + validatorsSeen := make(map[string]bool) + for _, path := range metadataPaths { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var metadata struct { + ValidatorName string `json:"validator_name"` + BatchIndex int `json:"batch_index"` + } + if err := json.Unmarshal(raw, &metadata); err != nil { + t.Fatal(err) + } + if metadata.ValidatorName == "" { + continue + } + if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 { + t.Fatalf("duplicate or split validation call: %+v", metadata) + } + validatorsSeen[metadata.ValidatorName] = true + } + if len(validatorsSeen) != validations { + t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations) + } + return requests +} + +func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) { + f := newProcessFixture(t, "decision-mixed", "[grammar]") + f.setSegments(t, "teh", "cat", "bad", "slow") + code, stdout, stderr := f.run(t, true, "--debug") + if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + output := f.readTranscript(t, f.output) + normalized, ledger := f.debugArtifacts(t) + assertSpanMetadata(t, normalized, output) + if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" { + t.Fatalf("partial validation transcript: %+v", output) + } + errors, recoveries := f.errors(t), f.recoveries(t) + if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 { + t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries) + } + if len(ledger) != 4 { + t.Fatalf("unexpected candidate ledger entries: %+v", ledger) + } + for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} { + entry := ledger[index] + var candidate int + var reason string + if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 { + t.Fatal(err) + } + if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) { + t.Fatalf("candidate %d evidence: %+v", index, entry) + } + if index > 0 { + var decisions []struct { + Disposition string `json:"disposition"` + } + if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition { + t.Fatalf("candidate %d decision: %+v %v", index, decisions, err) + } + } + } + var mixedSeen bool + for _, diagnostic := range f.validationDiagnostics(t) { + var reconciliation struct { + UnavailableIndices []int `json:"unavailable_indices"` + IgnoredIndices []int `json:"ignored_indices"` + Decisions []struct { + ProposalIndex int `json:"proposal_index"` + ReasonCode string `json:"reason_code"` + } `json:"decisions"` + } + if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) { + continue + } + var decoded struct { + Validations []struct { + CorrectionIndex int `json:"correction_index"` + } `json:"validations"` + } + if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil { + t.Fatal(err) + } + var finalContent string + if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" { + t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded) + } + mixedSeen = true + } + if !mixedSeen { + t.Fatal("missing mixed decision interaction evidence") + } + requests := f.assertDecisionRequestsUseOneAttempt(t) + full, survivor := 0, 0 + for _, indices := range requests { + switch { + case reflect.DeepEqual(indices, []int{0, 1, 2, 3}): + full++ + case reflect.DeepEqual(indices, []int{0}): + survivor++ + default: + t.Fatalf("unexpected validation identities: %v", requests) + } + } + if full != 1 || survivor == 0 { + t.Fatalf("wrong validation progression: %v", requests) + } +} + +func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) { + f := newProcessFixture(t, "decision-extra-only", "[grammar]") + f.setSegments(t, "teh") + code, stdout, stderr := f.run(t, true, "--debug") + if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + output := f.readTranscript(t, f.output) + if len(output.Segments) != 1 || output.Segments[0].Text != "the" { + t.Fatalf("lost usable correction: %+v", output) + } + errors, recoveries := f.errors(t), f.recoveries(t) + if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 { + t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries) + } + markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md")) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) { + t.Fatalf("Markdown totals disagree: %s", markdown) + } + _, ledger := f.debugArtifacts(t) + if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" { + t.Fatalf("unknown candidate in ledger: %+v", ledger) + } + ignored := 0 + for _, diagnostic := range f.validationDiagnostics(t) { + var reconciliation struct { + IgnoredIndices []int `json:"ignored_indices"` + } + if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) { + ignored++ + } + } + if ignored != 1 { + t.Fatalf("expected one handled extra decision, got %d", ignored) + } + requests := f.assertDecisionRequestsUseOneAttempt(t) + for _, indices := range requests { + if !reflect.DeepEqual(indices, []int{0}) { + t.Fatalf("unexpected identities: %v", requests) + } + } +} + func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) { f := newProcessFixture(t, "span-clean", "[spoken_word]") f.setSegments(t, "uh hello", "I I think") diff --git a/docs/internal/validators.md b/docs/internal/validators.md index e0b6e61..1b12fab 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -46,7 +46,9 @@ For each module section: 3. record decisions and compact operation outcomes; 4. carry only approved proposals forward. -Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator. +Decision cardinality is enforced inside Audita: each candidate proposal must +receive exactly one outcome per validator it reaches. The runner still rejects +duplicate, missing, or unknown identities in an internal validator result. ## LLM Validator Batching @@ -59,9 +61,23 @@ LLM validators: Oversized single inputs are rejected with a consolidated `validation_input_too_large` operation, without a provider call. Recoverable model failures make only their batch candidates unavailable while other -batches continue. Missing, duplicate, or unknown decision indices invalidate -that batch as a whole. Fatal errors stop the chain without returning unchecked -candidates as approved. Domain acceptance has no semantic retry. +batches continue. PromptKit validates each complete response against the +validator-decision JSON Schema and performs configured structural repair before +Audita receives decoded decisions. Audita then reconciles those decisions with +the correction indices requested in that batch: one usable entry keeps its +approval or editorial rejection; a missing or duplicate requested entry makes +only that candidate unavailable; unexpected indices are ignored. An empty list +makes all requested candidates unavailable. Results are ordered by request +identity and satisfy strict internal cardinality before the runner advances +approved candidates. An unavailable candidate never advances as approved. +Invalid confidence in a decoded test double also makes only its candidate +unavailable; production schema validation still rejects out-of-range confidence +before reconciliation. A partially unavailable batch records one failed +operation with only those requested indices, while a successful batch with +ignored extras records a decision-filtering recovery. Fatal errors stop the +chain without returning unchecked candidates as approved. Reconciliation makes +no semantic retry or extra model call; guided recovery remains +[deferred](../roadmap/future.md#domain-aware-response-recovery). ## Decision and Rejection Reporting diff --git a/docs/roadmap/decision-set-reconciliation.md b/docs/roadmap/decision-set-reconciliation.md index 432f79e..01b420d 100644 --- a/docs/roadmap/decision-set-reconciliation.md +++ b/docs/roadmap/decision-set-reconciliation.md @@ -2,8 +2,7 @@ **Last updated:** 2026-09-25 -**Status:** Accepted feature direction; implementation not started. Intended for -consideration before v2.0.0. This document owns the target behavior and user intent. +**Status:** Implemented. This document owns the target behavior and user intent. [implementation.md](implementation.md) owns concrete decisions, ordered work, and verification. @@ -19,7 +18,7 @@ Missing validation must never authorize applying a correction, but neither shoul it invalidate a different candidate's unambiguous approval or editorial rejection. The existing [resilience design](resilience.md) remains the foundation. -## Current Baseline +## Historical Baseline The reviewed application baseline is `d2a6fcf`: diff --git a/docs/roadmap/future.md b/docs/roadmap/future.md index 5765673..1527a11 100644 --- a/docs/roadmap/future.md +++ b/docs/roadmap/future.md @@ -9,22 +9,20 @@ intent; inclusion does not commit us to implementing every capability. ## Domain-Aware Response Recovery -First implement [partial success for validator decision sets](decision-set-reconciliation.md), -preserving usable decisions without another model call. Reconsider guided recovery -after that simpler change, based on the failures that remain. General per-entry +The [partial-success decision reconciliation](decision-set-reconciliation.md) +preserves usable decisions without another model call. Guided recovery remains +deferred pending evidence from failures that remain. General per-entry salvage from structurally invalid responses is also a separate design question; the accepted reconciliation feature retains whole-response structural validation. -Add bounded feedback and regeneration for model responses that satisfy the -structural schema but violate Audita's domain response contract, such as missing, -duplicate, or unexpected validator decision indices. The goal is to recover more -usable work before excluding affected candidates, reducing the impact of an -isolated malformed decision set. +Consider bounded feedback and regeneration for model responses that satisfy the +structural schema but leave requested decisions unavailable after reconciliation. +The goal would be to recover more usable work before excluding affected +candidates, when doing so materially improves outcomes. -This capability is desired but explicitly deferred from the current resilience -work. That work retains existing PromptKit structural repair and safe filtering; -unresolved domain-invalid decision sets produce errors and exclude their affected -candidates while other candidates can succeed. +This capability is explicitly deferred. Current behavior retains PromptKit +structural repair and per-candidate reconciliation; unavailable requested +decisions produce errors while other candidates can succeed. Before implementation, define the Audita/PromptKit ownership boundary, which checks are eligible for feedback, how feedback identifies defects without exposing diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d4fe91a..3db35ac 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ **Last updated:** 2026-09-25 -**Status:** Ready for implementation; no stages implemented by this planning pass. +**Status:** Implemented; all stages completed. **Source baseline:** `60e2fe7`. **Audience:** gpt-5.6-terra coding agent. @@ -230,6 +230,14 @@ success and handled extras through existing reporting mechanisms. ## Stage 3 — Verify the Assembled Workflow and Finalize Documentation +**Completed:** 2026-09-25. Controlled subprocess runs covered a mixed decision +response and an extra-only response through the CLI, transcript, YAML, Markdown, +debug ledger, and interaction evidence. Fixture execution facts showed one +provider attempt and zero structural repairs per logical call. `go test +./cmd/audita` and `go test ./...` passed. Canonical validator guidance and +roadmap status were updated; the two span-deletion plan links now target the +historical plan at `60e2fe7`. + **Work** 1. Extend the existing local-provider subprocess fixtures in diff --git a/docs/roadmap/span-deletion.md b/docs/roadmap/span-deletion.md index d705ece..11de91f 100644 --- a/docs/roadmap/span-deletion.md +++ b/docs/roadmap/span-deletion.md @@ -2,9 +2,10 @@ **Last updated:** 2026-09-25 -**Status:** Implemented. This document owns the desired -behavior and user intent. [implementation.md](implementation.md) owns the concrete -steps, selected implementation details, and completion evidence. +**Status:** Implemented. This document owns the desired behavior and user intent. +The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md) +records the concrete steps, selected implementation details, and completion +evidence. ## Purpose and User Intent @@ -197,5 +198,6 @@ in [future.md](future.md). This feature changes neither retry budgets nor the Audita/PromptKit ownership boundary and requires no upstream library feature. Implementation sequencing and concrete names belong to the -[implementation plan](implementation.md). The target remains consistent deletion -of spans within surviving segments, with no new operator controls or report types. +[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md). +The target remains consistent deletion of spans within surviving segments, with +no new operator controls or report types.