Verify partial validator decisions through CLI workflows

This commit is contained in:
2026-09-25 11:34:26 +00:00
parent 22f0932389
commit b8089f4a50
6 changed files with 361 additions and 35 deletions

View File

@@ -48,18 +48,23 @@ func TestProcessSubprocessHelper(t *testing.T) {
}
type fixtureProvider struct {
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
mode string
secret string
entered chan struct{}
beforeResponse func()
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
validationRequests [][]int
extraDecisionSent bool
mode string
secret string
entered chan struct{}
beforeResponse func()
}
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
var request struct {
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
ResponseFormat struct {
JSONSchema struct {
Schema struct {
@@ -73,6 +78,15 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
return
}
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
var requestedIndices []int
if validation && strings.HasPrefix(p.mode, "decision-") {
var err error
requestedIndices, err = fixtureValidationIndices(request.Messages)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
p.mu.Lock()
p.calls++
if validation {
@@ -82,6 +96,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
validationCall := p.validationCalls
proposalCall := p.proposalCalls
if validation && strings.HasPrefix(p.mode, "decision-") {
p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...))
}
addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent
if addExtra {
p.extraDecisionSent = true
}
beforeResponse := p.beforeResponse
p.mu.Unlock()
if beforeResponse != nil {
@@ -131,6 +152,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
}
}
if strings.HasPrefix(p.mode, "decision-") && !validation {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
if p.mode == "decision-mixed" {
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
}
}
if validation && !strings.HasPrefix(p.mode, "span-") {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "repair-filter-later-failure" {
@@ -143,11 +170,53 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
}
}
}
if validation && strings.HasPrefix(p.mode, "decision-") {
decisions := make([]map[string]any, 0, len(requestedIndices)+2)
for _, index := range requestedIndices {
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 {
continue
}
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1)
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
}
}
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
}
raw, _ := json.Marshal(map[string]any{"validations": decisions})
content = string(raw)
}
encoded, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
}
func fixtureValidationIndices(messages []struct {
Content string `json:"content"`
}) ([]int, error) {
for _, message := range messages {
const marker = "Corrections to validate:"
position := strings.LastIndex(message.Content, marker)
if position < 0 {
continue
}
var items []struct {
CorrectionIndex int `json:"correction_index"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
return nil, fmt.Errorf("decode validation request items: %w", err)
}
indices := make([]int, len(items))
for i, item := range items {
indices[i] = item.CorrectionIndex
}
return indices, nil
}
return nil, fmt.Errorf("validation request items not found")
}
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
type processFixture struct {
@@ -180,11 +249,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
repairs = 1
}
if strings.HasPrefix(mode, "span-") {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
if strings.HasPrefix(mode, "span-") {
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
}
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
@@ -365,6 +434,240 @@ func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
}
}
func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage {
t.Helper()
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json"))
if err != nil {
t.Fatal(err)
}
var diagnostics []map[string]json.RawMessage
for _, path := range paths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var payload map[string]json.RawMessage
if err := json.Unmarshal(raw, &payload); err != nil {
t.Fatal(err)
}
if _, validation := payload["reconciliation"]; validation {
diagnostics = append(diagnostics, payload)
}
}
return diagnostics
}
func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int {
t.Helper()
f.provider.mu.Lock()
requests := append([][]int(nil), f.provider.validationRequests...)
providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls
f.provider.mu.Unlock()
if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations {
t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests)
}
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
if err != nil || len(paths) != 1 {
t.Fatalf("execution path: %v %v", paths, err)
}
raw, err := os.ReadFile(paths[0])
if err != nil {
t.Fatal(err)
}
var execution struct {
Calls []struct {
Summary struct {
ProviderAttempts int `json:"provider_attempts"`
StructuralRepairs int `json:"structural_repairs"`
} `json:"summary"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &execution); err != nil {
t.Fatal(err)
}
if len(execution.Calls) != providerCalls {
t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls)
}
for _, call := range execution.Calls {
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
t.Fatalf("extra generation or structural repair: %+v", call)
}
}
metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json"))
if err != nil {
t.Fatal(err)
}
validatorsSeen := make(map[string]bool)
for _, path := range metadataPaths {
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var metadata struct {
ValidatorName string `json:"validator_name"`
BatchIndex int `json:"batch_index"`
}
if err := json.Unmarshal(raw, &metadata); err != nil {
t.Fatal(err)
}
if metadata.ValidatorName == "" {
continue
}
if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 {
t.Fatalf("duplicate or split validation call: %+v", metadata)
}
validatorsSeen[metadata.ValidatorName] = true
}
if len(validatorsSeen) != validations {
t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations)
}
return requests
}
func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
f := newProcessFixture(t, "decision-mixed", "[grammar]")
f.setSegments(t, "teh", "cat", "bad", "slow")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
normalized, ledger := f.debugArtifacts(t)
assertSpanMetadata(t, normalized, output)
if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" {
t.Fatalf("partial validation transcript: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries)
}
if len(ledger) != 4 {
t.Fatalf("unexpected candidate ledger entries: %+v", ledger)
}
for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} {
entry := ledger[index]
var candidate int
var reason string
if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 {
t.Fatal(err)
}
if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) {
t.Fatalf("candidate %d evidence: %+v", index, entry)
}
if index > 0 {
var decisions []struct {
Disposition string `json:"disposition"`
}
if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition {
t.Fatalf("candidate %d decision: %+v %v", index, decisions, err)
}
}
}
var mixedSeen bool
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
UnavailableIndices []int `json:"unavailable_indices"`
IgnoredIndices []int `json:"ignored_indices"`
Decisions []struct {
ProposalIndex int `json:"proposal_index"`
ReasonCode string `json:"reason_code"`
} `json:"decisions"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) {
continue
}
var decoded struct {
Validations []struct {
CorrectionIndex int `json:"correction_index"`
} `json:"validations"`
}
if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil {
t.Fatal(err)
}
var finalContent string
if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" {
t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded)
}
mixedSeen = true
}
if !mixedSeen {
t.Fatal("missing mixed decision interaction evidence")
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
full, survivor := 0, 0
for _, indices := range requests {
switch {
case reflect.DeepEqual(indices, []int{0, 1, 2, 3}):
full++
case reflect.DeepEqual(indices, []int{0}):
survivor++
default:
t.Fatalf("unexpected validation identities: %v", requests)
}
}
if full != 1 || survivor == 0 {
t.Fatalf("wrong validation progression: %v", requests)
}
}
func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
f := newProcessFixture(t, "decision-extra-only", "[grammar]")
f.setSegments(t, "teh")
code, stdout, stderr := f.run(t, true, "--debug")
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output := f.readTranscript(t, f.output)
if len(output.Segments) != 1 || output.Segments[0].Text != "the" {
t.Fatalf("lost usable correction: %+v", output)
}
errors, recoveries := f.errors(t), f.recoveries(t)
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries)
}
markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
if err != nil {
t.Fatal(err)
}
if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) {
t.Fatalf("Markdown totals disagree: %s", markdown)
}
_, ledger := f.debugArtifacts(t)
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" {
t.Fatalf("unknown candidate in ledger: %+v", ledger)
}
ignored := 0
for _, diagnostic := range f.validationDiagnostics(t) {
var reconciliation struct {
IgnoredIndices []int `json:"ignored_indices"`
}
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
t.Fatal(err)
}
if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) {
ignored++
}
}
if ignored != 1 {
t.Fatalf("expected one handled extra decision, got %d", ignored)
}
requests := f.assertDecisionRequestsUseOneAttempt(t)
for _, indices := range requests {
if !reflect.DeepEqual(indices, []int{0}) {
t.Fatalf("unexpected identities: %v", requests)
}
}
}
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
f := newProcessFixture(t, "span-clean", "[spoken_word]")
f.setSegments(t, "uh hello", "I I think")

View File

@@ -46,7 +46,9 @@ For each module section:
3. record decisions and compact operation outcomes;
4. carry only approved proposals forward.
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
Decision cardinality is enforced inside Audita: each candidate proposal must
receive exactly one outcome per validator it reaches. The runner still rejects
duplicate, missing, or unknown identities in an internal validator result.
## LLM Validator Batching
@@ -59,9 +61,23 @@ LLM validators:
Oversized single inputs are rejected with a consolidated
`validation_input_too_large` operation, without a provider call. Recoverable
model failures make only their batch candidates unavailable while other
batches continue. Missing, duplicate, or unknown decision indices invalidate
that batch as a whole. Fatal errors stop the chain without returning unchecked
candidates as approved. Domain acceptance has no semantic retry.
batches continue. PromptKit validates each complete response against the
validator-decision JSON Schema and performs configured structural repair before
Audita receives decoded decisions. Audita then reconciles those decisions with
the correction indices requested in that batch: one usable entry keeps its
approval or editorial rejection; a missing or duplicate requested entry makes
only that candidate unavailable; unexpected indices are ignored. An empty list
makes all requested candidates unavailable. Results are ordered by request
identity and satisfy strict internal cardinality before the runner advances
approved candidates. An unavailable candidate never advances as approved.
Invalid confidence in a decoded test double also makes only its candidate
unavailable; production schema validation still rejects out-of-range confidence
before reconciliation. A partially unavailable batch records one failed
operation with only those requested indices, while a successful batch with
ignored extras records a decision-filtering recovery. Fatal errors stop the
chain without returning unchecked candidates as approved. Reconciliation makes
no semantic retry or extra model call; guided recovery remains
[deferred](../roadmap/future.md#domain-aware-response-recovery).
## Decision and Rejection Reporting

View File

@@ -2,8 +2,7 @@
**Last updated:** 2026-09-25
**Status:** Accepted feature direction; implementation not started. Intended for
consideration before v2.0.0. This document owns the target behavior and user intent.
**Status:** Implemented. This document owns the target behavior and user intent.
[implementation.md](implementation.md) owns concrete decisions, ordered work,
and verification.
@@ -19,7 +18,7 @@ Missing validation must never authorize applying a correction, but neither shoul
it invalidate a different candidate's unambiguous approval or editorial rejection.
The existing [resilience design](resilience.md) remains the foundation.
## Current Baseline
## Historical Baseline
The reviewed application baseline is `d2a6fcf`:

View File

@@ -9,22 +9,20 @@ intent; inclusion does not commit us to implementing every capability.
## Domain-Aware Response Recovery
First implement [partial success for validator decision sets](decision-set-reconciliation.md),
preserving usable decisions without another model call. Reconsider guided recovery
after that simpler change, based on the failures that remain. General per-entry
The [partial-success decision reconciliation](decision-set-reconciliation.md)
preserves usable decisions without another model call. Guided recovery remains
deferred pending evidence from failures that remain. General per-entry
salvage from structurally invalid responses is also a separate design question;
the accepted reconciliation feature retains whole-response structural validation.
Add bounded feedback and regeneration for model responses that satisfy the
structural schema but violate Audita's domain response contract, such as missing,
duplicate, or unexpected validator decision indices. The goal is to recover more
usable work before excluding affected candidates, reducing the impact of an
isolated malformed decision set.
Consider bounded feedback and regeneration for model responses that satisfy the
structural schema but leave requested decisions unavailable after reconciliation.
The goal would be to recover more usable work before excluding affected
candidates, when doing so materially improves outcomes.
This capability is desired but explicitly deferred from the current resilience
work. That work retains existing PromptKit structural repair and safe filtering;
unresolved domain-invalid decision sets produce errors and exclude their affected
candidates while other candidates can succeed.
This capability is explicitly deferred. Current behavior retains PromptKit
structural repair and per-candidate reconciliation; unavailable requested
decisions produce errors while other candidates can succeed.
Before implementation, define the Audita/PromptKit ownership boundary, which
checks are eligible for feedback, how feedback identifies defects without exposing

View File

@@ -2,7 +2,7 @@
**Last updated:** 2026-09-25
**Status:** Ready for implementation; no stages implemented by this planning pass.
**Status:** Implemented; all stages completed.
**Source baseline:** `60e2fe7`.
**Audience:** gpt-5.6-terra coding agent.
@@ -230,6 +230,14 @@ success and handled extras through existing reporting mechanisms.
## Stage 3 — Verify the Assembled Workflow and Finalize Documentation
**Completed:** 2026-09-25. Controlled subprocess runs covered a mixed decision
response and an extra-only response through the CLI, transcript, YAML, Markdown,
debug ledger, and interaction evidence. Fixture execution facts showed one
provider attempt and zero structural repairs per logical call. `go test
./cmd/audita` and `go test ./...` passed. Canonical validator guidance and
roadmap status were updated; the two span-deletion plan links now target the
historical plan at `60e2fe7`.
**Work**
1. Extend the existing local-provider subprocess fixtures in

View File

@@ -2,9 +2,10 @@
**Last updated:** 2026-09-25
**Status:** Implemented. This document owns the desired
behavior and user intent. [implementation.md](implementation.md) owns the concrete
steps, selected implementation details, and completion evidence.
**Status:** Implemented. This document owns the desired behavior and user intent.
The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md)
records the concrete steps, selected implementation details, and completion
evidence.
## Purpose and User Intent
@@ -197,5 +198,6 @@ in [future.md](future.md). This feature changes neither retry budgets nor the
Audita/PromptKit ownership boundary and requires no upstream library feature.
Implementation sequencing and concrete names belong to the
[implementation plan](implementation.md). The target remains consistent deletion
of spans within surviving segments, with no new operator controls or report types.
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md).
The target remains consistent deletion of spans within surviving segments, with
no new operator controls or report types.