Verify partial validator decisions through CLI workflows
This commit is contained in:
@@ -48,18 +48,23 @@ func TestProcessSubprocessHelper(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fixtureProvider struct {
|
type fixtureProvider struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
calls int
|
calls int
|
||||||
proposalCalls int
|
proposalCalls int
|
||||||
validationCalls int
|
validationCalls int
|
||||||
mode string
|
validationRequests [][]int
|
||||||
secret string
|
extraDecisionSent bool
|
||||||
entered chan struct{}
|
mode string
|
||||||
beforeResponse func()
|
secret string
|
||||||
|
entered chan struct{}
|
||||||
|
beforeResponse func()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||||
var request struct {
|
var request struct {
|
||||||
|
Messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
} `json:"messages"`
|
||||||
ResponseFormat struct {
|
ResponseFormat struct {
|
||||||
JSONSchema struct {
|
JSONSchema struct {
|
||||||
Schema struct {
|
Schema struct {
|
||||||
@@ -73,6 +78,15 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
||||||
|
var requestedIndices []int
|
||||||
|
if validation && strings.HasPrefix(p.mode, "decision-") {
|
||||||
|
var err error
|
||||||
|
requestedIndices, err = fixtureValidationIndices(request.Messages)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
p.calls++
|
p.calls++
|
||||||
if validation {
|
if validation {
|
||||||
@@ -82,6 +96,13 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
validationCall := p.validationCalls
|
validationCall := p.validationCalls
|
||||||
proposalCall := p.proposalCalls
|
proposalCall := p.proposalCalls
|
||||||
|
if validation && strings.HasPrefix(p.mode, "decision-") {
|
||||||
|
p.validationRequests = append(p.validationRequests, append([]int(nil), requestedIndices...))
|
||||||
|
}
|
||||||
|
addExtra := p.mode == "decision-extra-only" && validation && !p.extraDecisionSent
|
||||||
|
if addExtra {
|
||||||
|
p.extraDecisionSent = true
|
||||||
|
}
|
||||||
beforeResponse := p.beforeResponse
|
beforeResponse := p.beforeResponse
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
if beforeResponse != nil {
|
if beforeResponse != nil {
|
||||||
@@ -131,6 +152,12 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.HasPrefix(p.mode, "decision-") && !validation {
|
||||||
|
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99}]}`
|
||||||
|
if p.mode == "decision-mixed" {
|
||||||
|
content = `{"corrections":[{"id":1,"original_text":"teh","corrected_text":"the","confidence":0.99},{"id":2,"original_text":"cat","corrected_text":"dog","confidence":0.99},{"id":3,"original_text":"bad","corrected_text":"good","confidence":0.99},{"id":4,"original_text":"slow","corrected_text":"fast","confidence":0.99}]}`
|
||||||
|
}
|
||||||
|
}
|
||||||
if validation && !strings.HasPrefix(p.mode, "span-") {
|
if validation && !strings.HasPrefix(p.mode, "span-") {
|
||||||
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||||
if p.mode == "repair-filter-later-failure" {
|
if p.mode == "repair-filter-later-failure" {
|
||||||
@@ -143,11 +170,53 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if validation && strings.HasPrefix(p.mode, "decision-") {
|
||||||
|
decisions := make([]map[string]any, 0, len(requestedIndices)+2)
|
||||||
|
for _, index := range requestedIndices {
|
||||||
|
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 2 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
approved := !(p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 1)
|
||||||
|
decisions = append(decisions, map[string]any{"correction_index": index, "approved": approved, "confidence": 0.99, "reason": "controlled review"})
|
||||||
|
if p.mode == "decision-mixed" && len(requestedIndices) == 4 && index == 3 {
|
||||||
|
decisions = append(decisions, map[string]any{"correction_index": index, "approved": false, "confidence": 0.99, "reason": "conflicting review"})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if p.mode == "decision-mixed" && len(requestedIndices) == 4 || addExtra {
|
||||||
|
decisions = append(decisions, map[string]any{"correction_index": 99, "approved": true, "confidence": 0.99, "reason": "unexpected review"})
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(map[string]any{"validations": decisions})
|
||||||
|
content = string(raw)
|
||||||
|
}
|
||||||
encoded, _ := json.Marshal(content)
|
encoded, _ := json.Marshal(content)
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fixtureValidationIndices(messages []struct {
|
||||||
|
Content string `json:"content"`
|
||||||
|
}) ([]int, error) {
|
||||||
|
for _, message := range messages {
|
||||||
|
const marker = "Corrections to validate:"
|
||||||
|
position := strings.LastIndex(message.Content, marker)
|
||||||
|
if position < 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var items []struct {
|
||||||
|
CorrectionIndex int `json:"correction_index"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(strings.TrimSpace(message.Content[position+len(marker):])), &items); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode validation request items: %w", err)
|
||||||
|
}
|
||||||
|
indices := make([]int, len(items))
|
||||||
|
for i, item := range items {
|
||||||
|
indices[i] = item.CorrectionIndex
|
||||||
|
}
|
||||||
|
return indices, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("validation request items not found")
|
||||||
|
}
|
||||||
|
|
||||||
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
|
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
|
||||||
|
|
||||||
type processFixture struct {
|
type processFixture struct {
|
||||||
@@ -180,11 +249,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|||||||
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
||||||
repairs = 1
|
repairs = 1
|
||||||
}
|
}
|
||||||
if strings.HasPrefix(mode, "span-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
repairs = 1
|
repairs = 1
|
||||||
}
|
}
|
||||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
||||||
if strings.HasPrefix(mode, "span-") {
|
if strings.HasPrefix(mode, "span-") || strings.HasPrefix(mode, "decision-") {
|
||||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
||||||
@@ -365,6 +434,240 @@ func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (f processFixture) validationDiagnostics(t *testing.T) []map[string]json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-response-payload.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var diagnostics []map[string]json.RawMessage
|
||||||
|
for _, path := range paths {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var payload map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, validation := payload["reconciliation"]; validation {
|
||||||
|
diagnostics = append(diagnostics, payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return diagnostics
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f processFixture) assertDecisionRequestsUseOneAttempt(t *testing.T) [][]int {
|
||||||
|
t.Helper()
|
||||||
|
f.provider.mu.Lock()
|
||||||
|
requests := append([][]int(nil), f.provider.validationRequests...)
|
||||||
|
providerCalls, proposals, validations := f.provider.calls, f.provider.proposalCalls, f.provider.validationCalls
|
||||||
|
f.provider.mu.Unlock()
|
||||||
|
if proposals != 1 || validations != len(requests) || validations == 0 || providerCalls != proposals+validations {
|
||||||
|
t.Fatalf("provider requests: total=%d proposals=%d validations=%d identities=%v", providerCalls, proposals, validations, requests)
|
||||||
|
}
|
||||||
|
paths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "llm-execution.json"))
|
||||||
|
if err != nil || len(paths) != 1 {
|
||||||
|
t.Fatalf("execution path: %v %v", paths, err)
|
||||||
|
}
|
||||||
|
raw, err := os.ReadFile(paths[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var execution struct {
|
||||||
|
Calls []struct {
|
||||||
|
Summary struct {
|
||||||
|
ProviderAttempts int `json:"provider_attempts"`
|
||||||
|
StructuralRepairs int `json:"structural_repairs"`
|
||||||
|
} `json:"summary"`
|
||||||
|
} `json:"calls"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &execution); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(execution.Calls) != providerCalls {
|
||||||
|
t.Fatalf("logical/provider calls: %d/%d", len(execution.Calls), providerCalls)
|
||||||
|
}
|
||||||
|
for _, call := range execution.Calls {
|
||||||
|
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
|
||||||
|
t.Fatalf("extra generation or structural repair: %+v", call)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
metadataPaths, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*", "*", "*-request-metadata.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
validatorsSeen := make(map[string]bool)
|
||||||
|
for _, path := range metadataPaths {
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var metadata struct {
|
||||||
|
ValidatorName string `json:"validator_name"`
|
||||||
|
BatchIndex int `json:"batch_index"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &metadata); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if metadata.ValidatorName == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if validatorsSeen[metadata.ValidatorName] || metadata.BatchIndex != 0 {
|
||||||
|
t.Fatalf("duplicate or split validation call: %+v", metadata)
|
||||||
|
}
|
||||||
|
validatorsSeen[metadata.ValidatorName] = true
|
||||||
|
}
|
||||||
|
if len(validatorsSeen) != validations {
|
||||||
|
t.Fatalf("validator operations=%d requests=%d", len(validatorsSeen), validations)
|
||||||
|
}
|
||||||
|
return requests
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessKeepsUsableValidatorDecisionsInMixedResponse(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "decision-mixed", "[grammar]")
|
||||||
|
f.setSegments(t, "teh", "cat", "bad", "slow")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--debug")
|
||||||
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 0 recoveries") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
output := f.readTranscript(t, f.output)
|
||||||
|
normalized, ledger := f.debugArtifacts(t)
|
||||||
|
assertSpanMetadata(t, normalized, output)
|
||||||
|
if len(output.Segments) != 4 || output.Segments[0].Text != "the" || output.Segments[1].Text != "cat" || output.Segments[2].Text != "bad" || output.Segments[3].Text != "slow" {
|
||||||
|
t.Fatalf("partial validation transcript: %+v", output)
|
||||||
|
}
|
||||||
|
errors, recoveries := f.errors(t), f.recoveries(t)
|
||||||
|
if errors.Status != reporting.StatusCompleted || len(errors.Errors) != 1 || len(errors.Errors[0].Failures) != 1 || errors.Errors[0].Failures[0].Reason != "validation_decisions_invalid" || !reflect.DeepEqual(errors.Errors[0].Failures[0].ProposalIndices, []int{2, 3}) || *errors.Summary.CorrectionsApplied != 1 || *errors.Summary.WorkUnitsWithErrors != 1 || *errors.Summary.Recoveries != 0 || len(recoveries.Recoveries) != 0 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
|
||||||
|
t.Fatalf("partial validation reports: errors=%+v recoveries=%+v", errors, recoveries)
|
||||||
|
}
|
||||||
|
if len(ledger) != 4 {
|
||||||
|
t.Fatalf("unexpected candidate ledger entries: %+v", ledger)
|
||||||
|
}
|
||||||
|
for index, want := range []struct{ disposition, reason string }{{"applied", ""}, {"rejected", "llm_rejected"}, {"unavailable", "missing_validation_decision"}, {"unavailable", "duplicate_validation_decision"}} {
|
||||||
|
entry := ledger[index]
|
||||||
|
var candidate int
|
||||||
|
var reason string
|
||||||
|
if err := json.Unmarshal(entry["proposal_index"], &candidate); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(entry["disposition_reason_code"], &reason); err != nil && len(entry["disposition_reason_code"]) > 0 {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if candidate != index || ledgerDisposition(entry) != want.disposition || reason != want.reason || (index != 0 && entry["applied_corrected_text"] != nil) {
|
||||||
|
t.Fatalf("candidate %d evidence: %+v", index, entry)
|
||||||
|
}
|
||||||
|
if index > 0 {
|
||||||
|
var decisions []struct {
|
||||||
|
Disposition string `json:"disposition"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(entry["llm_validator_decisions"], &decisions); err != nil || len(decisions) != 1 || decisions[0].Disposition != want.disposition {
|
||||||
|
t.Fatalf("candidate %d decision: %+v %v", index, decisions, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var mixedSeen bool
|
||||||
|
for _, diagnostic := range f.validationDiagnostics(t) {
|
||||||
|
var reconciliation struct {
|
||||||
|
UnavailableIndices []int `json:"unavailable_indices"`
|
||||||
|
IgnoredIndices []int `json:"ignored_indices"`
|
||||||
|
Decisions []struct {
|
||||||
|
ProposalIndex int `json:"proposal_index"`
|
||||||
|
ReasonCode string `json:"reason_code"`
|
||||||
|
} `json:"decisions"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(reconciliation.UnavailableIndices, []int{2, 3}) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var decoded struct {
|
||||||
|
Validations []struct {
|
||||||
|
CorrectionIndex int `json:"correction_index"`
|
||||||
|
} `json:"validations"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(diagnostic["decoded"], &decoded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var finalContent string
|
||||||
|
if err := json.Unmarshal(diagnostic["final_content"], &finalContent); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) || len(decoded.Validations) != 5 || len(reconciliation.Decisions) != 4 || reconciliation.Decisions[2].ProposalIndex != 2 || reconciliation.Decisions[2].ReasonCode != "missing_validation_decision" || reconciliation.Decisions[3].ProposalIndex != 3 || reconciliation.Decisions[3].ReasonCode != "duplicate_validation_decision" || finalContent == "" {
|
||||||
|
t.Fatalf("mixed response evidence: %+v %+v", reconciliation, decoded)
|
||||||
|
}
|
||||||
|
mixedSeen = true
|
||||||
|
}
|
||||||
|
if !mixedSeen {
|
||||||
|
t.Fatal("missing mixed decision interaction evidence")
|
||||||
|
}
|
||||||
|
requests := f.assertDecisionRequestsUseOneAttempt(t)
|
||||||
|
full, survivor := 0, 0
|
||||||
|
for _, indices := range requests {
|
||||||
|
switch {
|
||||||
|
case reflect.DeepEqual(indices, []int{0, 1, 2, 3}):
|
||||||
|
full++
|
||||||
|
case reflect.DeepEqual(indices, []int{0}):
|
||||||
|
survivor++
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected validation identities: %v", requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if full != 1 || survivor == 0 {
|
||||||
|
t.Fatalf("wrong validation progression: %v", requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSubprocessReportsFilteredExtraValidatorDecision(t *testing.T) {
|
||||||
|
f := newProcessFixture(t, "decision-extra-only", "[grammar]")
|
||||||
|
f.setSegments(t, "teh")
|
||||||
|
code, stdout, stderr := f.run(t, true, "--debug")
|
||||||
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 1 recoveries") {
|
||||||
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||||
|
}
|
||||||
|
output := f.readTranscript(t, f.output)
|
||||||
|
if len(output.Segments) != 1 || output.Segments[0].Text != "the" {
|
||||||
|
t.Fatalf("lost usable correction: %+v", output)
|
||||||
|
}
|
||||||
|
errors, recoveries := f.errors(t), f.recoveries(t)
|
||||||
|
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 1 || !reflect.DeepEqual(recoveries.Recoveries[0].Reasons, []string{"validation_decisions_filtered"}) || recoveries.Recoveries[0].Operation != "validation" || len(recoveries.Recoveries[0].FilteredProposals) != 0 || *recoveries.Summary.CorrectionsApplied != 1 || *recoveries.Summary.Recoveries != 1 || *recoveries.Summary.ProposalResponsesWithFiltering != 0 {
|
||||||
|
t.Fatalf("extra-only reports: errors=%+v recoveries=%+v", errors, recoveries)
|
||||||
|
}
|
||||||
|
markdown, err := os.ReadFile(filepath.Join(f.dir, "report.md"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(markdown, []byte("Corrections applied: 1")) || !bytes.Contains(markdown, []byte("Errors: 0 work units")) || !bytes.Contains(markdown, []byte("Recoveries: 1 of ")) {
|
||||||
|
t.Fatalf("Markdown totals disagree: %s", markdown)
|
||||||
|
}
|
||||||
|
_, ledger := f.debugArtifacts(t)
|
||||||
|
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "applied" {
|
||||||
|
t.Fatalf("unknown candidate in ledger: %+v", ledger)
|
||||||
|
}
|
||||||
|
ignored := 0
|
||||||
|
for _, diagnostic := range f.validationDiagnostics(t) {
|
||||||
|
var reconciliation struct {
|
||||||
|
IgnoredIndices []int `json:"ignored_indices"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(diagnostic["reconciliation"], &reconciliation); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if reflect.DeepEqual(reconciliation.IgnoredIndices, []int{99}) {
|
||||||
|
ignored++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ignored != 1 {
|
||||||
|
t.Fatalf("expected one handled extra decision, got %d", ignored)
|
||||||
|
}
|
||||||
|
requests := f.assertDecisionRequestsUseOneAttempt(t)
|
||||||
|
for _, indices := range requests {
|
||||||
|
if !reflect.DeepEqual(indices, []int{0}) {
|
||||||
|
t.Fatalf("unexpected identities: %v", requests)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
|
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
|
||||||
f := newProcessFixture(t, "span-clean", "[spoken_word]")
|
f := newProcessFixture(t, "span-clean", "[spoken_word]")
|
||||||
f.setSegments(t, "uh hello", "I I think")
|
f.setSegments(t, "uh hello", "I I think")
|
||||||
|
|||||||
@@ -46,7 +46,9 @@ For each module section:
|
|||||||
3. record decisions and compact operation outcomes;
|
3. record decisions and compact operation outcomes;
|
||||||
4. carry only approved proposals forward.
|
4. carry only approved proposals forward.
|
||||||
|
|
||||||
Decision cardinality is enforced: each candidate proposal must receive exactly one decision per validator.
|
Decision cardinality is enforced inside Audita: each candidate proposal must
|
||||||
|
receive exactly one outcome per validator it reaches. The runner still rejects
|
||||||
|
duplicate, missing, or unknown identities in an internal validator result.
|
||||||
|
|
||||||
## LLM Validator Batching
|
## LLM Validator Batching
|
||||||
|
|
||||||
@@ -59,9 +61,23 @@ LLM validators:
|
|||||||
Oversized single inputs are rejected with a consolidated
|
Oversized single inputs are rejected with a consolidated
|
||||||
`validation_input_too_large` operation, without a provider call. Recoverable
|
`validation_input_too_large` operation, without a provider call. Recoverable
|
||||||
model failures make only their batch candidates unavailable while other
|
model failures make only their batch candidates unavailable while other
|
||||||
batches continue. Missing, duplicate, or unknown decision indices invalidate
|
batches continue. PromptKit validates each complete response against the
|
||||||
that batch as a whole. Fatal errors stop the chain without returning unchecked
|
validator-decision JSON Schema and performs configured structural repair before
|
||||||
candidates as approved. Domain acceptance has no semantic retry.
|
Audita receives decoded decisions. Audita then reconciles those decisions with
|
||||||
|
the correction indices requested in that batch: one usable entry keeps its
|
||||||
|
approval or editorial rejection; a missing or duplicate requested entry makes
|
||||||
|
only that candidate unavailable; unexpected indices are ignored. An empty list
|
||||||
|
makes all requested candidates unavailable. Results are ordered by request
|
||||||
|
identity and satisfy strict internal cardinality before the runner advances
|
||||||
|
approved candidates. An unavailable candidate never advances as approved.
|
||||||
|
Invalid confidence in a decoded test double also makes only its candidate
|
||||||
|
unavailable; production schema validation still rejects out-of-range confidence
|
||||||
|
before reconciliation. A partially unavailable batch records one failed
|
||||||
|
operation with only those requested indices, while a successful batch with
|
||||||
|
ignored extras records a decision-filtering recovery. Fatal errors stop the
|
||||||
|
chain without returning unchecked candidates as approved. Reconciliation makes
|
||||||
|
no semantic retry or extra model call; guided recovery remains
|
||||||
|
[deferred](../roadmap/future.md#domain-aware-response-recovery).
|
||||||
|
|
||||||
## Decision and Rejection Reporting
|
## Decision and Rejection Reporting
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,7 @@
|
|||||||
|
|
||||||
**Last updated:** 2026-09-25
|
**Last updated:** 2026-09-25
|
||||||
|
|
||||||
**Status:** Accepted feature direction; implementation not started. Intended for
|
**Status:** Implemented. This document owns the target behavior and user intent.
|
||||||
consideration before v2.0.0. This document owns the target behavior and user intent.
|
|
||||||
[implementation.md](implementation.md) owns concrete decisions, ordered work,
|
[implementation.md](implementation.md) owns concrete decisions, ordered work,
|
||||||
and verification.
|
and verification.
|
||||||
|
|
||||||
@@ -19,7 +18,7 @@ Missing validation must never authorize applying a correction, but neither shoul
|
|||||||
it invalidate a different candidate's unambiguous approval or editorial rejection.
|
it invalidate a different candidate's unambiguous approval or editorial rejection.
|
||||||
The existing [resilience design](resilience.md) remains the foundation.
|
The existing [resilience design](resilience.md) remains the foundation.
|
||||||
|
|
||||||
## Current Baseline
|
## Historical Baseline
|
||||||
|
|
||||||
The reviewed application baseline is `d2a6fcf`:
|
The reviewed application baseline is `d2a6fcf`:
|
||||||
|
|
||||||
|
|||||||
@@ -9,22 +9,20 @@ intent; inclusion does not commit us to implementing every capability.
|
|||||||
|
|
||||||
## Domain-Aware Response Recovery
|
## Domain-Aware Response Recovery
|
||||||
|
|
||||||
First implement [partial success for validator decision sets](decision-set-reconciliation.md),
|
The [partial-success decision reconciliation](decision-set-reconciliation.md)
|
||||||
preserving usable decisions without another model call. Reconsider guided recovery
|
preserves usable decisions without another model call. Guided recovery remains
|
||||||
after that simpler change, based on the failures that remain. General per-entry
|
deferred pending evidence from failures that remain. General per-entry
|
||||||
salvage from structurally invalid responses is also a separate design question;
|
salvage from structurally invalid responses is also a separate design question;
|
||||||
the accepted reconciliation feature retains whole-response structural validation.
|
the accepted reconciliation feature retains whole-response structural validation.
|
||||||
|
|
||||||
Add bounded feedback and regeneration for model responses that satisfy the
|
Consider bounded feedback and regeneration for model responses that satisfy the
|
||||||
structural schema but violate Audita's domain response contract, such as missing,
|
structural schema but leave requested decisions unavailable after reconciliation.
|
||||||
duplicate, or unexpected validator decision indices. The goal is to recover more
|
The goal would be to recover more usable work before excluding affected
|
||||||
usable work before excluding affected candidates, reducing the impact of an
|
candidates, when doing so materially improves outcomes.
|
||||||
isolated malformed decision set.
|
|
||||||
|
|
||||||
This capability is desired but explicitly deferred from the current resilience
|
This capability is explicitly deferred. Current behavior retains PromptKit
|
||||||
work. That work retains existing PromptKit structural repair and safe filtering;
|
structural repair and per-candidate reconciliation; unavailable requested
|
||||||
unresolved domain-invalid decision sets produce errors and exclude their affected
|
decisions produce errors while other candidates can succeed.
|
||||||
candidates while other candidates can succeed.
|
|
||||||
|
|
||||||
Before implementation, define the Audita/PromptKit ownership boundary, which
|
Before implementation, define the Audita/PromptKit ownership boundary, which
|
||||||
checks are eligible for feedback, how feedback identifies defects without exposing
|
checks are eligible for feedback, how feedback identifies defects without exposing
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
**Last updated:** 2026-09-25
|
**Last updated:** 2026-09-25
|
||||||
|
|
||||||
**Status:** Ready for implementation; no stages implemented by this planning pass.
|
**Status:** Implemented; all stages completed.
|
||||||
**Source baseline:** `60e2fe7`.
|
**Source baseline:** `60e2fe7`.
|
||||||
**Audience:** gpt-5.6-terra coding agent.
|
**Audience:** gpt-5.6-terra coding agent.
|
||||||
|
|
||||||
@@ -230,6 +230,14 @@ success and handled extras through existing reporting mechanisms.
|
|||||||
|
|
||||||
## Stage 3 — Verify the Assembled Workflow and Finalize Documentation
|
## Stage 3 — Verify the Assembled Workflow and Finalize Documentation
|
||||||
|
|
||||||
|
**Completed:** 2026-09-25. Controlled subprocess runs covered a mixed decision
|
||||||
|
response and an extra-only response through the CLI, transcript, YAML, Markdown,
|
||||||
|
debug ledger, and interaction evidence. Fixture execution facts showed one
|
||||||
|
provider attempt and zero structural repairs per logical call. `go test
|
||||||
|
./cmd/audita` and `go test ./...` passed. Canonical validator guidance and
|
||||||
|
roadmap status were updated; the two span-deletion plan links now target the
|
||||||
|
historical plan at `60e2fe7`.
|
||||||
|
|
||||||
**Work**
|
**Work**
|
||||||
|
|
||||||
1. Extend the existing local-provider subprocess fixtures in
|
1. Extend the existing local-provider subprocess fixtures in
|
||||||
|
|||||||
@@ -2,9 +2,10 @@
|
|||||||
|
|
||||||
**Last updated:** 2026-09-25
|
**Last updated:** 2026-09-25
|
||||||
|
|
||||||
**Status:** Implemented. This document owns the desired
|
**Status:** Implemented. This document owns the desired behavior and user intent.
|
||||||
behavior and user intent. [implementation.md](implementation.md) owns the concrete
|
The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md)
|
||||||
steps, selected implementation details, and completion evidence.
|
records the concrete steps, selected implementation details, and completion
|
||||||
|
evidence.
|
||||||
|
|
||||||
## Purpose and User Intent
|
## Purpose and User Intent
|
||||||
|
|
||||||
@@ -197,5 +198,6 @@ in [future.md](future.md). This feature changes neither retry budgets nor the
|
|||||||
Audita/PromptKit ownership boundary and requires no upstream library feature.
|
Audita/PromptKit ownership boundary and requires no upstream library feature.
|
||||||
|
|
||||||
Implementation sequencing and concrete names belong to the
|
Implementation sequencing and concrete names belong to the
|
||||||
[implementation plan](implementation.md). The target remains consistent deletion
|
[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/60e2fe7/docs/roadmap/implementation.md).
|
||||||
of spans within surviving segments, with no new operator controls or report types.
|
The target remains consistent deletion of spans within surviving segments, with
|
||||||
|
no new operator controls or report types.
|
||||||
|
|||||||
Reference in New Issue
Block a user