diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index 8c8979c..ce4879b 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "os/signal" "path/filepath" + "reflect" "strconv" "strings" "sync" @@ -20,6 +21,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/cli" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" + "gitea.maximumdirect.net/eric/audita/internal/core/schema" "gopkg.in/yaml.v3" ) @@ -106,7 +108,30 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}` } } - if validation { + if strings.HasPrefix(p.mode, "span-") { + switch p.mode { + case "span-clean", "span-filter": + content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}` + if p.mode == "span-filter" { + content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}` + } + case "span-spacing": + content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}` + case "span-negative": + content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}` + } + if validation { + if p.mode == "span-clean" || p.mode == "span-filter" { + content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` + } else { + content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` + } + if p.mode == "span-negative" && validationCall == 2 { + content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}` + } + } + } + if validation && !strings.HasPrefix(p.mode, "span-") { content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` if p.mode == "repair-filter-later-failure" { content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` @@ -155,7 +180,13 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" { repairs = 1 } + if strings.HasPrefix(mode, "span-") { + repairs = 1 + } configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs")) + if strings.HasPrefix(mode, "span-") { + configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n" + } if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil { t.Fatal(err) } @@ -208,6 +239,219 @@ func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument { return document } +func (f processFixture) recoveries(t *testing.T) reporting.RecoveriesDocument { + t.Helper() + raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")) + if err != nil { + t.Fatal(err) + } + var document reporting.RecoveriesDocument + if err := yaml.Unmarshal(raw, &document); err != nil { + t.Fatal(err) + } + return document +} + +func (f processFixture) setSegments(t *testing.T, texts ...string) { + t.Helper() + segments := make([]schema.Segment, len(texts)) + for i, value := range texts { + segments[i] = schema.Segment{ID: i + 1, Speaker: "A", Start: float64(i * 10), End: float64(i*10 + 1), Text: value, Categories: []string{"dialogue"}} + } + raw, err := json.Marshal(segments) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(f.transcript, raw, 0600); err != nil { + t.Fatal(err) + } +} + +func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transcript { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + transcript, err := schema.ParseTranscriptJSON(raw) + if err != nil { + t.Fatal(err) + } + return transcript +} + +func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) { + t.Helper() + dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) + if err != nil || len(dirs) != 1 { + t.Fatalf("expected one debug run: %v %v", dirs, err) + } + normalized := f.readTranscript(t, filepath.Join(dirs[0], "normalized-transcript.json")) + raw, err := os.ReadFile(filepath.Join(dirs[0], "correction-ledger.json")) + if err != nil { + t.Fatal(err) + } + var ledger []map[string]json.RawMessage + if err := json.Unmarshal(raw, &ledger); err != nil { + t.Fatal(err) + } + return normalized, ledger +} + +func assertSpanMetadata(t *testing.T, normalized, output *schema.Transcript) { + t.Helper() + if len(normalized.Segments) != len(output.Segments) { + t.Fatalf("segment membership changed: normalized=%+v output=%+v", normalized, output) + } + for i := range normalized.Segments { + normalized.Segments[i].Text = output.Segments[i].Text + } + if !reflect.DeepEqual(normalized, output) { + t.Fatalf("segment metadata/order changed: normalized=%+v output=%+v", normalized, output) + } +} + +func ledgerDisposition(entry map[string]json.RawMessage) string { + var value string + _ = json.Unmarshal(entry["disposition"], &value) + return value +} + +func ledgerHasDecision(t *testing.T, entry map[string]json.RawMessage, field, key string, approved bool) bool { + t.Helper() + var decisions []struct { + ValidatorKey string `json:"validator_key"` + Approved bool `json:"approved"` + } + if err := json.Unmarshal(entry[field], &decisions); err != nil { + t.Fatal(err) + } + for _, decision := range decisions { + if decision.ValidatorKey == key && decision.Approved == approved { + return true + } + } + return false +} + +func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) { + t.Helper() + dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) + if err != nil || len(dirs) != 1 { + t.Fatalf("debug run: %v %v", dirs, err) + } + raw, err := os.ReadFile(filepath.Join(dirs[0], "llm-execution.json")) + if err != nil { + t.Fatal(err) + } + var execution struct { + Calls []struct { + Summary struct { + ProviderAttempts int `json:"provider_attempts"` + StructuralRepairs int `json:"structural_repairs"` + } `json:"summary"` + } `json:"calls"` + } + if err := json.Unmarshal(raw, &execution); err != nil { + t.Fatal(err) + } + if len(execution.Calls) != wantCalls || f.provider.callCount() != wantCalls { + t.Fatalf("unexpected generation counts: calls=%d provider=%d", len(execution.Calls), f.provider.callCount()) + } + for _, call := range execution.Calls { + if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 { + t.Fatalf("unexpected structural repair: %+v", call) + } + } +} + +func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) { + f := newProcessFixture(t, "span-clean", "[spoken_word]") + f.setSegments(t, "uh hello", "I I think") + code, stdout, stderr := f.run(t, true, "--debug") + if code != 0 || stdout != "" || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 0 recoveries") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + output := f.readTranscript(t, f.output) + normalized, ledger := f.debugArtifacts(t) + assertSpanMetadata(t, normalized, output) + if len(output.Segments) != 2 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" { + t.Fatalf("unexpected corrected transcript: %+v", output) + } + if len(ledger) != 2 || ledgerDisposition(ledger[0]) != "applied" || ledgerDisposition(ledger[1]) != "applied" || string(ledger[0]["proposed_corrected_text"]) != `""` || string(ledger[0]["applied_corrected_text"]) != `""` { + t.Fatalf("deletion missing from ledger: %+v", ledger) + } + errors, recoveries := f.errors(t), f.recoveries(t) + if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 0 || *errors.Summary.CorrectionsApplied != 2 || *errors.Summary.WorkUnitsCompleted != 1 || *recoveries.Summary.CorrectionsApplied != 2 { + t.Fatalf("report totals disagree: errors=%+v recoveries=%+v", errors, recoveries) + } + f.assertNoStructuralRepairs(t, 3) +} + +func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) { + f := newProcessFixture(t, "span-spacing", "[grammar]") + f.setSegments(t, "hello world") + code, _, stderr := f.run(t, true, "--debug") + if code != 0 || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") { + t.Fatalf("code=%d stderr=%q", code, stderr) + } + output := f.readTranscript(t, f.output) + normalized, _ := f.debugArtifacts(t) + assertSpanMetadata(t, normalized, output) + if len(output.Segments) != 1 || output.Segments[0].Text != "hello world" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 { + t.Fatalf("spacing correction failed: %+v", output) + } + f.assertNoStructuralRepairs(t, 3) +} + +func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) { + f := newProcessFixture(t, "span-filter", "[spoken_word]") + f.setSegments(t, "uh hello", "I I think", "uh") + code, _, stderr := f.run(t, true, "--debug") + if code != 0 || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 1 recoveries") { + t.Fatalf("code=%d stderr=%q", code, stderr) + } + output := f.readTranscript(t, f.output) + normalized, ledger := f.debugArtifacts(t) + assertSpanMetadata(t, normalized, output) + if len(output.Segments) != 3 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" || output.Segments[2].Text != "uh" { + t.Fatalf("filtered segment or survivors wrong: %+v", output) + } + recoveries := f.recoveries(t) + if len(f.errors(t).Errors) != 0 || len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].Reasons) != 1 || recoveries.Recoveries[0].Reasons[0] != "proposals_filtered" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || recoveries.Recoveries[0].FilteredProposals[0].Reason != "empty_resulting_segment" || *recoveries.Summary.ProposalResponsesWithFiltering != 1 { + t.Fatalf("filtering recovery wrong: %+v", recoveries) + } + if len(ledger) != 3 || ledgerDisposition(ledger[2]) != "rejected" || string(ledger[2]["proposed_corrected_text"]) != `""` { + t.Fatalf("filtered deletion evidence missing: %+v", ledger) + } + if !ledgerHasDecision(t, ledger[2], "deterministic_validator_decisions", "non_empty_resulting_segment", false) { + t.Fatalf("renamed validator missing from evidence: %+v", ledger[2]) + } + f.assertNoStructuralRepairs(t, 3) +} + +func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) { + f := newProcessFixture(t, "span-negative", "[grammar]") + f.setSegments(t, "not good") + code, _, stderr := f.run(t, true, "--debug") + if code != 0 || !strings.Contains(stderr, "0 corrections applied; 0 work units with errors; 0 recoveries") { + t.Fatalf("code=%d stderr=%q", code, stderr) + } + output := f.readTranscript(t, f.output) + normalized, ledger := f.debugArtifacts(t) + assertSpanMetadata(t, normalized, output) + if len(output.Segments) != 1 || output.Segments[0].Text != "not good" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 { + t.Fatalf("negative review changed outcome: %+v", output) + } + if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "rejected" || string(ledger[0]["proposed_corrected_text"]) != `""` { + t.Fatalf("rejection evidence missing: %+v", ledger) + } + if _, present := ledger[0]["applied_corrected_text"]; present || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "editorial_review", true) || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "meaning_reversal_review", false) { + t.Fatalf("negative decision not recorded: %+v", ledger[0]) + } + f.assertNoStructuralRepairs(t, 3) +} + func TestSubprocessDefaultFileOutputAndReports(t *testing.T) { f := newProcessFixture(t, "normal", "[grammar]") code, stdout, stderr := f.run(t, true) diff --git a/docs/internal/prompts.md b/docs/internal/prompts.md index 8a547f7..a196e7b 100644 --- a/docs/internal/prompts.md +++ b/docs/internal/prompts.md @@ -38,6 +38,6 @@ path. Prepared runtime metadata belongs in diagnostics rather than the registry. The structured-call contract carries prompt ID/version, string variables, response-schema key, and logical-call attribution. Module and validator callers -are being migrated in their respective work units. See the +pass these values through the role-bound adapter. See the [LLM runtime](llm-runtime.md) for runtime assembly and [configuration](../config.md) for operator-selected profiles. diff --git a/docs/migrations/v2.md b/docs/migrations/v2.md index 656e9ca..da97cdf 100644 --- a/docs/migrations/v2.md +++ b/docs/migrations/v2.md @@ -125,3 +125,11 @@ capacity and usage fields retain known/complete flags. Debug runs write material. See [operations](../operations.md#diagnostics-directory) for placement and paths, [configuration](../config.md) for all settings, and the [CLI reference](../cli.md) for current flags. + +Debug ledger consumers should recognize the deterministic validator key +`non_empty_resulting_segment` in place of `non_empty_corrected_text`. The +`empty_resulting_segment` reason is unchanged. Candidate-backed ledger entries +now include `proposed_corrected_text: ""` for an empty replacement, and applied +deletions also include `applied_corrected_text: ""`; omitted fields still mean +there was no candidate or no applied edit, respectively. See the +[debug artifact contract](../integrations/output-files.md#debug-only-files). diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index 16efa50..13b1e94 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ **Last updated:** 2026-09-25 -**Status:** In progress; Stages 1–3 complete. +**Status:** Implemented; all stages complete. **Source baseline:** `a83a518` (feature roadmap); application baseline `0b6cd98`. **Audience:** gpt-5.6-terra coding agent. @@ -12,7 +12,8 @@ Implement [Consistent Span Deletion](span-deletion.md). That document owns the product intent, target semantics, and acceptance criteria. This document owns the concrete decisions, ordered work, and validation. Execute Stage 1 through Stage 4 in order. Record actual completion and checks beneath each stage when implemented. -Do not interpret this ready status as implemented behavior. +The completion evidence below records the implemented behavior; the source +baseline remains historical. Read [development guidance](../development.md) and all [agent](../policy/agents.md), [architecture](../policy/architecture.md), [documentation](../policy/documentation.md), @@ -351,6 +352,13 @@ schema/adapter tests, domain/application tests, serialized debug tests, and representative complete CLI workflows. The workspace is committed and the docs describe implemented behavior without claiming whole-segment removal. +**Completed 2026-09-25.** Four offline CLI workflows verified clean deletion, +literal spacing, partial success with blank-result filtering, and an ordinary +negative meaning-review decision. Metadata was checked against normalized +input; the existing runner test covers fixed section membership and order. +`go test ./...` passed. Documentation references, affected links, and whitespace +were checked. Whole-segment removal remains deferred. + ## Open Questions None requiring a product decision remain. Validator names, schema/prompt identity, diff --git a/docs/roadmap/library-adoption.md b/docs/roadmap/library-adoption.md index 744c5f6..9c41eb6 100644 --- a/docs/roadmap/library-adoption.md +++ b/docs/roadmap/library-adoption.md @@ -4,8 +4,8 @@ **Status:** Production implementation complete. ConfigKit v0.4.0 and PromptKit v0.11.0 are adopted; [historical implementation evidence](https://gitea.maximumdirect.net/eric/audita/src/commit/cb405f6/docs/roadmap/implementation.md) records the -completed checks. The current `implementation.md` is a separate resilience -roadmap. Release compliance and publication remain separate work. +completed checks. The current `implementation.md` covers span deletion instead. +Release compliance and publication remain separate work. ## Goal and Ownership diff --git a/docs/roadmap/remediation.md b/docs/roadmap/remediation.md index 15b4b9b..5dfc27a 100644 --- a/docs/roadmap/remediation.md +++ b/docs/roadmap/remediation.md @@ -3,7 +3,7 @@ **Reviewed:** 2026-09-25, source commit `ec78b5a`. **Scope:** Review of the resilience implementation for unintentionally strengthened invariants, contracts, guarantees, and test requirements, compared with -[resilience.md](resilience.md), [implementation.md](implementation.md), and the +[resilience.md](resilience.md), [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md), and the [agent policy](../policy/agents.md). **Status:** R11–R14 addressed. The findings below describe the reviewed source; see the resolution record for the implemented changes and verification. diff --git a/docs/roadmap/resilience.md b/docs/roadmap/resilience.md index 4182516..70b027f 100644 --- a/docs/roadmap/resilience.md +++ b/docs/roadmap/resilience.md @@ -4,7 +4,7 @@ **Status:** Implemented. This document records the selected behavior and user intent. Concrete design decisions and completion evidence are in -[implementation.md](implementation.md). +[historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md). ## Purpose and User Intent @@ -283,7 +283,7 @@ volume, and a short reason breakdown. Detailed inspection belongs in YAML or deb ## Implementation Reference -The product decisions are settled. [implementation.md](implementation.md) defines +The product decisions are settled. The [historical implementation plan](https://gitea.maximumdirect.net/eric/audita/src/commit/a83a518/docs/roadmap/implementation.md) defines the selected settings and precedence, YAML schemas, counts and ordering, interrupted work presentation, destination checks, and output finalization. It also owns the ordered implementation stages and validation work. This document remains the diff --git a/docs/roadmap/span-deletion.md b/docs/roadmap/span-deletion.md index 117281b..d705ece 100644 --- a/docs/roadmap/span-deletion.md +++ b/docs/roadmap/span-deletion.md @@ -2,7 +2,7 @@ **Last updated:** 2026-09-25 -**Status:** Accepted target; not implemented. This document owns the desired +**Status:** Implemented. This document owns the desired behavior and user intent. [implementation.md](implementation.md) owns the concrete steps, selected implementation details, and completion evidence. @@ -19,7 +19,7 @@ authorize whole-segment removal, broaden module responsibilities, or introduce new recovery machinery. The accepted [resilience behavior](resilience.md) remains in effect, including partial success, fixed chunks, and validation before mutation. -## Current Baseline +## Historical Baseline (`0b6cd98`) The source reviewed at `0b6cd98` contains a mismatch: