diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go index cb3b84b..55eb315 100644 --- a/cmd/audita/main_integration_test.go +++ b/cmd/audita/main_integration_test.go @@ -9,10 +9,13 @@ import ( "net/http/httptest" "os" "os/exec" + "os/signal" "path/filepath" "strings" "sync" + "syscall" "testing" + "time" "gitea.maximumdirect.net/eric/audita/internal/cli" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" @@ -25,7 +28,10 @@ func TestProcessSubprocessHelper(t *testing.T) { } for i, arg := range os.Args { if arg == "--" { - os.Exit(cli.Run(context.Background(), os.Args[i+1:], os.Stdout, os.Stderr)) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr) + stop() + os.Exit(code) } } os.Exit(2) @@ -34,9 +40,11 @@ func TestProcessSubprocessHelper(t *testing.T) { type fixtureProvider struct { mu sync.Mutex calls int + proposalCalls int validationCalls int mode string secret string + entered chan struct{} } func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { @@ -58,17 +66,44 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) { p.calls++ if validation { p.validationCalls++ + } else { + p.proposalCalls++ } validationCall := p.validationCalls + proposalCall := p.proposalCalls p.mu.Unlock() - if p.mode == "all-model-failure" || (p.mode == "partial-validation" && validation && validationCall == 1) { + if p.mode == "wait-cancel" { + select { + case p.entered <- struct{}{}: + default: + } + <-r.Context().Done() + return + } + if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) { w.WriteHeader(http.StatusInternalServerError) _, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret) return } content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}` + if p.mode == "repair-filter-later-failure" && !validation { + if proposalCall == 1 { + content = `{"wrong":true}` + } else { + content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}` + } + } if validation { content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` + if p.mode == "repair-filter-later-failure" { + content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}` + } else if p.mode == "invalid-decision-after-repair" { + if validationCall == 1 { + content = `{"wrong":true}` + } else { + content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}` + } + } } encoded, _ := json.Marshal(content) w.Header().Set("Content-Type", "application/json") @@ -90,7 +125,7 @@ type processFixture struct { func newProcessFixture(t *testing.T, mode, modules string) processFixture { t.Helper() dir := t.TempDir() - p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`} + p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)} server := httptest.NewServer(http.HandlerFunc(p.serve)) t.Cleanup(server.Close) if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil { @@ -103,7 +138,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture { } } configPath := filepath.Join(dir, "config.yml") - configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, modules, filepath.Join(dir, "runs")) + repairs := 0 + if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" { + repairs = 1 + } + configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs")) if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil { t.Fatal(err) } @@ -170,9 +209,13 @@ func TestSubprocessDefaultFileOutputAndReports(t *testing.T) { t.Fatalf("errors document: %+v", document) } for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { - if _, err := os.Stat(filepath.Join(f.dir, name)); err != nil { + report, err := os.ReadFile(filepath.Join(f.dir, name)) + if err != nil { t.Fatal(err) } + if bytes.Contains(report, []byte("hello world")) || bytes.Contains(report, []byte("Hello world")) || bytes.Contains(report, []byte(f.provider.secret)) { + t.Fatalf("ordinary report contains payload or secret: %s", name) + } } if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) { t.Fatalf("debug created by default: %v", err) @@ -198,24 +241,43 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) { } func TestSubprocessReportingAndDebugSelections(t *testing.T) { - for _, debug := range []bool{false, true} { + var expectedOutput, expectedSummary string + for _, selection := range []struct{ reporting, debug bool }{ + {false, false}, {false, true}, {true, false}, {true, true}, + } { f := newProcessFixture(t, "normal", "[grammar]") extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")} - if debug { + if selection.reporting { + extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")} + } + if selection.debug { extra = append(extra, "--debug") } code, stdout, stderr := f.run(t, true, extra...) if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") { - t.Fatalf("debug=%v code=%d stdout=%q stderr=%q", debug, code, stdout, stderr) + t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr) } - if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) { - t.Fatalf("disabled report created: %v", err) + output, err := os.ReadFile(f.output) + if err != nil { + t.Fatal(err) + } + summary := strings.SplitN(stderr, "\n", 2)[0] + if expectedOutput == "" { + expectedOutput, expectedSummary = string(output), summary + } else if string(output) != expectedOutput || summary != expectedSummary { + t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output) + } + for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} { + _, err := os.Stat(filepath.Join(f.dir, name)) + if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) { + t.Fatalf("selection=%+v report %s: %v", selection, name, err) + } } dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*")) - if err != nil || len(dirs) != boolInt(debug) { - t.Fatalf("debug=%v dirs=%v %v", debug, dirs, err) + if err != nil || len(dirs) != boolInt(selection.debug) { + t.Fatalf("selection=%+v dirs=%v %v", selection, dirs, err) } - if debug { + if selection.debug { for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} { if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil { t.Fatal(err) @@ -233,9 +295,9 @@ func boolInt(value bool) int { } func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { - f := newProcessFixture(t, "all-model-failure", "[grammar]") + f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]") code, stdout, stderr := f.run(t, true) - if code != 0 || stdout != "" || !strings.Contains(stderr, "1 work units with errors") { + if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) } output, err := os.ReadFile(f.output) @@ -243,7 +305,7 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { t.Fatalf("normalized output: %s %v", output, err) } document := f.errors(t) - if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 1 { + if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 { t.Fatalf("model failure report: %+v", document) } if strings.Contains(stderr, f.provider.secret) { @@ -251,6 +313,100 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) { } } +func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) { + f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]") + code, stdout, stderr := f.run(t, true) + if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") { + t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr) + } + output, err := os.ReadFile(f.output) + if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) { + t.Fatalf("unchecked output: %s %v", output, err) + } + document := f.errors(t) + if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 4 { + t.Fatalf("validation failure report: %+v; calls=%d", document, f.provider.callCount()) + } +} + +func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) { + f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]") + code, _, stderr := f.run(t, true) + if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") { + t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t)) + } + output, err := os.ReadFile(f.output) + if err != nil || !bytes.Contains(output, []byte("Hello world")) { + t.Fatalf("output: %s %v", output, err) + } + errors := f.errors(t) + if len(errors.Errors) != 1 || errors.Errors[0].ModuleInstance != "grammar_2" { + t.Fatalf("later failure: %+v", errors) + } + raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")) + if err != nil { + t.Fatal(err) + } + var recoveries reporting.RecoveriesDocument + if err := yaml.Unmarshal(raw, &recoveries); err != nil { + t.Fatal(err) + } + if len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].ModuleInstance != "grammar_1" || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || *recoveries.Summary.Recoveries != 1 { + t.Fatalf("combined recovery: %+v", recoveries) + } + if !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "output_repaired") || !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "proposals_filtered") { + t.Fatalf("recovery reasons: %+v", recoveries.Recoveries[0]) + } +} + +func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) { + f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]") + code, _, stderr := f.run(t, true) + if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") { + t.Fatalf("code=%d stderr=%q", code, stderr) + } + document := f.errors(t) + if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || len(document.Errors[0].Failures) != 1 || document.Errors[0].Failures[0].Reason != "validation_decisions_invalid" { + t.Fatalf("invalid decisions report: %+v", document) + } + output, err := os.ReadFile(f.output) + if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) { + t.Fatalf("unchecked output: %s %v", output, err) + } +} + +func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T) { + f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]") + ctx, stop := context.WithTimeout(context.Background(), 5*time.Second) + defer stop() + args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output} + command := exec.CommandContext(ctx, os.Args[0], args...) + command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) + var stderr bytes.Buffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + select { + case <-f.provider.entered: + case <-ctx.Done(): + t.Fatalf("provider was not reached: %v", ctx.Err()) + } + if err := command.Process.Signal(os.Interrupt); err != nil { + t.Fatal(err) + } + err := command.Wait() + if ctx.Err() != nil || err == nil { + t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String()) + } + if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 { + t.Fatalf("cancellation report: %+v", document) + } + if _, err := os.Stat(f.output); !os.IsNotExist(err) { + t.Fatalf("canceled run published transcript: %v", err) + } +} + func TestSubprocessPartialValidationAndLaterStage(t *testing.T) { f := newProcessFixture(t, "partial-validation", "[grammar, grammar]") code, _, stderr := f.run(t, true) diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index eec3e31..533e22e 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,8 +2,8 @@ **Last updated:** 2026-09-24 -**Status:** Ready for implementation; all stages below are pending. Planning only: -no implementation is claimed by this document. +**Status:** Implemented. All stages below are complete; completion notes record +the checks performed for each unit of work. ## Scope and Execution Instructions @@ -823,6 +823,27 @@ competing current contract or implicit legacy compatibility promise. ## Stage 14 — Verify the Integrated Resilience Contract +**Completed 2026-09-24.** Audited the target acceptance criteria against the +assembled CLI, runner, validator, reporting, and output-session tests. Added +controlled-provider subprocess coverage for all-proposal and all-validation +failure across repeated modules, all reporting/debug selections, a repaired and +filtered proposal response followed by a later failure, a repaired response with +invalid validation decisions, and signal cancellation with failed reports and no +transcript. Added an atomic-replacement cleanup check for a failed rename and +asserted ordinary reports omit transcript text and the configured secret. Existing +runner/validator tests cover survivor re-batching, fixed chunk membership and +cross-chunk rejection after text growth, segment metadata and ordering, and +active/queued cancellation; output-session tests cover initial +and final write failures, intentional replacement, and destination alias checks. + +**Final validation:** `go test ./...` and `go vet ./...` passed. Focused +`go test -race` passed for runner, LLM, validators, process-report assembly, +reporting, diagnostics/output, and the subprocess package. The release build +script produced valid Linux amd64 and arm64 binaries using `v0.0.0` in a temporary +directory; a separate macOS arm64 cross-build also passed. No live provider or +released artifact was used; provider behavior is covered by deterministic local +HTTP fixtures. Builds were cross-compiled and not executed on target platforms. + **Work:** Audit every acceptance criterion in `resilience.md` against final code and retained tests. Add only missing integration coverage. In particular verify the following cross-component outcomes rather than relying solely on green leaf tests: diff --git a/docs/roadmap/resilience.md b/docs/roadmap/resilience.md index a996f29..4182516 100644 --- a/docs/roadmap/resilience.md +++ b/docs/roadmap/resilience.md @@ -2,9 +2,9 @@ **Last updated:** 2026-09-24 -**Status:** Target behavior agreed; not implemented. This document defines the -desired end state and user intent. Concrete design decisions and implementation -steps are recorded in [implementation.md](implementation.md). +**Status:** Implemented. This document records the selected behavior and user +intent. Concrete design decisions and completion evidence are in +[implementation.md](implementation.md). ## Purpose and User Intent @@ -33,20 +33,21 @@ Internal structures and measurements must serve Audita's processing workflow. The reporting package adapts those outcomes to structured and human-facing presentations, rather than reshaping processing around a reporting convenience. -## Current Baseline and Intended Change +## Historical Baseline and Implemented Change The [pipeline](../internal/pipeline.md), [LLM integration](../integrations/openai-compatible-llm.md), [report assembly](../internal/diagnostics-reporting.md), and [artifact lifecycle](../operations.md) describe current behavior. -Currently, exhausted structural repairs can become warnings and skipped proposals -or rejected validation batches. Other model runtime failures cancel peer work, -stop subsequent modules, and prevent final transcript output. Successful repairs -appear in execution observations rather than a dedicated recovery classification. -Detailed artifacts are written during ordinary runs and may subsequently be -removed by retention. The working transcript is re-chunked between modules. +Before this work, exhausted structural repairs could become warnings and skipped +proposals or rejected validation batches. Other model runtime failures canceled +peer work, stopped subsequent modules, and prevented final transcript output. +Successful repairs appeared in execution observations rather than a dedicated +recovery classification. Detailed artifacts were written during ordinary runs +and could later be removed by retention. The working transcript was re-chunked +between modules. -This feature deliberately changes those behaviors. It builds on the completed +This feature changed those behaviors. It builds on the completed [library adoption](library-adoption.md) and [review remediation](remediation.md). Fatal cancellation remains necessary; unsuccessful model work must be separated from failures that actually require cancellation of the run. @@ -278,8 +279,7 @@ volume, and a short reason breakdown. Detailed inspection belongs in YAML or deb failures, and reporting/debug selections. Preserve domain safeguards, capacity limits, and redaction. - Canonical CLI, configuration, operations, integration, internal, and architecture - documentation describes the implemented final behavior. Until implementation, - this roadmap does not change their current implemented contracts. + documentation describes the implemented final behavior. ## Implementation Reference diff --git a/internal/core/diagnostics/output_session_test.go b/internal/core/diagnostics/output_session_test.go index 546bec1..61377e7 100644 --- a/internal/core/diagnostics/output_session_test.go +++ b/internal/core/diagnostics/output_session_test.go @@ -83,6 +83,26 @@ func TestPrepareOutputPreservesExistingTranscriptAndChecksParent(t *testing.T) { } } +func TestAtomicReplacementCleansTemporaryFileAfterRenameFailure(t *testing.T) { + dir := t.TempDir() + blocked := filepath.Join(dir, "directory-target") + if err := os.Mkdir(blocked, 0700); err != nil { + t.Fatal(err) + } + if err := atomicReplace(blocked, []byte("new content")); err == nil { + t.Fatal("directory replacement unexpectedly succeeded") + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".audita-write-") { + t.Fatalf("replacement left temporary file: %s", entry.Name()) + } + } +} + func TestDisabledReportingIgnoresUnusableDirectory(t *testing.T) { dir := t.TempDir() session, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, "out.json"), ReportDir: filepath.Join(dir, "missing"), ReportingEnabled: false}) diff --git a/internal/framework/runner/runner_test.go b/internal/framework/runner/runner_test.go index 22b5186..2c3d357 100644 --- a/internal/framework/runner/runner_test.go +++ b/internal/framework/runner/runner_test.go @@ -279,7 +279,10 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) { cfg.Chunking.MaxSectionTokens = 3 cfg.Chunking.MinSectionTokens = 0 transcript := &schema.Transcript{Segments: []schema.Segment{ - {ID: 1, Text: "a"}, {ID: 2, Text: "b"}, {ID: 3, Text: "c"}, {ID: 4, Text: "d"}, + {ID: 1, Speaker: "A", Start: 0, End: 1, Text: "a"}, + {ID: 2, Speaker: "B", Start: 1, End: 2, Text: "b"}, + {ID: 3, Speaker: "A", Start: 2, End: 3, Text: "c"}, + {ID: 4, Speaker: "B", Start: 3, End: 4, Text: "d"}, }} var mu sync.Mutex seen := make(map[int][]int) @@ -302,6 +305,9 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) { if req.Section.Index == 0 && req.WorkingTranscript.Segments[0].Text != updatedText { return nil, errors.New("later module did not see updated section text") } + if req.Section.Index == 0 { + return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C", Confidence: 1}}, nil + } return nil, nil }}, }}) @@ -315,9 +321,15 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) { if len(seen) != 2 || len(seen[0]) != 2 || seen[0][0] != 1 || seen[0][1] != 2 || len(seen[1]) != 2 || seen[1][0] != 3 || seen[1][1] != 4 { t.Fatalf("later module saw changed section membership: %+v", seen) } - if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[3].ID != 4 { + if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[2].Text != "c" || out.FinalTranscript.Segments[3].ID != 4 { t.Fatalf("recombined transcript: %+v", out.FinalTranscript) } + for i, segment := range out.FinalTranscript.Segments { + original := transcript.Segments[i] + if segment.ID != original.ID || segment.Speaker != original.Speaker || segment.Start != original.Start || segment.End != original.End { + t.Fatalf("segment %d metadata changed: %+v", i, segment) + } + } if out.WorkUnits[0].CorrectionsApplied != 1 || out.WorkUnits[1].CorrectionsApplied != 0 { t.Fatalf("section application counts: %+v", out.WorkUnits) }