eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Verify the integrated resilient processing contract

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 18:03:25 +00:00
parent a770c77a23
commit 6523d7ac74
5 changed files with 242 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
cmd/audita/main_integration_test.go
View File

@@ -9,10 +9,13 @@ import (
"net/http/httptest"
"os"
"os/exec"
"os/signal"
"path/filepath"
"strings"
"sync"
"syscall"
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/cli"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
@@ -25,7 +28,10 @@ func TestProcessSubprocessHelper(t *testing.T) {
}
for i, arg := range os.Args {
if arg == "--" {
os.Exit(cli.Run(context.Background(), os.Args[i+1:], os.Stdout, os.Stderr))
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
stop()
os.Exit(code)
}
}
os.Exit(2)
@@ -34,9 +40,11 @@ func TestProcessSubprocessHelper(t *testing.T) {
type fixtureProvider struct {
mu sync.Mutex
calls int
proposalCalls int
validationCalls int
mode string
secret string
entered chan struct{}
}
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
@@ -58,17 +66,44 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
p.calls++
if validation {
p.validationCalls++
} else {
p.proposalCalls++
}
validationCall := p.validationCalls
proposalCall := p.proposalCalls
p.mu.Unlock()
if p.mode == "all-model-failure" || (p.mode == "partial-validation" && validation && validationCall == 1) {
if p.mode == "wait-cancel" {
select {
case p.entered <- struct{}{}:
default:
}
<-r.Context().Done()
return
}
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
return
}
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
if p.mode == "repair-filter-later-failure" && !validation {
if proposalCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
}
}
if validation {
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
if p.mode == "repair-filter-later-failure" {
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
} else if p.mode == "invalid-decision-after-repair" {
if validationCall == 1 {
content = `{"wrong":true}`
} else {
content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}`
}
}
}
encoded, _ := json.Marshal(content)
w.Header().Set("Content-Type", "application/json")
@@ -90,7 +125,7 @@ type processFixture struct {
func newProcessFixture(t *testing.T, mode, modules string) processFixture {
t.Helper()
dir := t.TempDir()
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`}
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
server := httptest.NewServer(http.HandlerFunc(p.serve))
t.Cleanup(server.Close)
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
@@ -103,7 +138,11 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
}
}
configPath := filepath.Join(dir, "config.yml")
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, modules, filepath.Join(dir, "runs"))
repairs := 0
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
repairs = 1
}
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
t.Fatal(err)
}
@@ -170,9 +209,13 @@ func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
t.Fatalf("errors document: %+v", document)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
if _, err := os.Stat(filepath.Join(f.dir, name)); err != nil {
report, err := os.ReadFile(filepath.Join(f.dir, name))
if err != nil {
t.Fatal(err)
}
if bytes.Contains(report, []byte("hello world")) || bytes.Contains(report, []byte("Hello world")) || bytes.Contains(report, []byte(f.provider.secret)) {
t.Fatalf("ordinary report contains payload or secret: %s", name)
}
}
if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
t.Fatalf("debug created by default: %v", err)
@@ -198,24 +241,43 @@ func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
}
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
for _, debug := range []bool{false, true} {
var expectedOutput, expectedSummary string
for _, selection := range []struct{ reporting, debug bool }{
{false, false}, {false, true}, {true, false}, {true, true},
} {
f := newProcessFixture(t, "normal", "[grammar]")
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
if debug {
if selection.reporting {
extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")}
}
if selection.debug {
extra = append(extra, "--debug")
}
code, stdout, stderr := f.run(t, true, extra...)
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
t.Fatalf("debug=%v code=%d stdout=%q stderr=%q", debug, code, stdout, stderr)
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
}
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
t.Fatalf("disabled report created: %v", err)
output, err := os.ReadFile(f.output)
if err != nil {
t.Fatal(err)
}
summary := strings.SplitN(stderr, "\n", 2)[0]
if expectedOutput == "" {
expectedOutput, expectedSummary = string(output), summary
} else if string(output) != expectedOutput || summary != expectedSummary {
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
}
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
_, err := os.Stat(filepath.Join(f.dir, name))
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
}
}
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
if err != nil || len(dirs) != boolInt(debug) {
t.Fatalf("debug=%v dirs=%v %v", debug, dirs, err)
if err != nil || len(dirs) != boolInt(selection.debug) {
t.Fatalf("selection=%+v dirs=%v %v", selection, dirs, err)
}
if debug {
if selection.debug {
for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} {
if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil {
t.Fatal(err)
@@ -233,9 +295,9 @@ func boolInt(value bool) int {
}
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-model-failure", "[grammar]")
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 work units with errors") {
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
@@ -243,7 +305,7 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
t.Fatalf("normalized output: %s %v", output, err)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 1 {
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
t.Fatalf("model failure report: %+v", document)
}
if strings.Contains(stderr, f.provider.secret) {
@@ -251,6 +313,100 @@ func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
}
}
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
code, stdout, stderr := f.run(t, true)
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("unchecked output: %s %v", output, err)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 4 {
t.Fatalf("validation failure report: %+v; calls=%d", document, f.provider.callCount())
}
}
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
code, _, stderr := f.run(t, true)
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("output: %s %v", output, err)
}
errors := f.errors(t)
if len(errors.Errors) != 1 || errors.Errors[0].ModuleInstance != "grammar_2" {
t.Fatalf("later failure: %+v", errors)
}
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
if err != nil {
t.Fatal(err)
}
var recoveries reporting.RecoveriesDocument
if err := yaml.Unmarshal(raw, &recoveries); err != nil {
t.Fatal(err)
}
if len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].ModuleInstance != "grammar_1" || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || *recoveries.Summary.Recoveries != 1 {
t.Fatalf("combined recovery: %+v", recoveries)
}
if !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "output_repaired") || !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "proposals_filtered") {
t.Fatalf("recovery reasons: %+v", recoveries.Recoveries[0])
}
}
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
code, _, stderr := f.run(t, true)
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
t.Fatalf("code=%d stderr=%q", code, stderr)
}
document := f.errors(t)
if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || len(document.Errors[0].Failures) != 1 || document.Errors[0].Failures[0].Reason != "validation_decisions_invalid" {
t.Fatalf("invalid decisions report: %+v", document)
}
output, err := os.ReadFile(f.output)
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
t.Fatalf("unchecked output: %s %v", output, err)
}
}
func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T) {
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
command := exec.CommandContext(ctx, os.Args[0], args...)
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
var stderr bytes.Buffer
command.Stderr = &stderr
if err := command.Start(); err != nil {
t.Fatal(err)
}
select {
case <-f.provider.entered:
case <-ctx.Done():
t.Fatalf("provider was not reached: %v", ctx.Err())
}
if err := command.Process.Signal(os.Interrupt); err != nil {
t.Fatal(err)
}
err := command.Wait()
if ctx.Err() != nil || err == nil {
t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String())
}
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
t.Fatalf("cancellation report: %+v", document)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("canceled run published transcript: %v", err)
}
}
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
code, _, stderr := f.run(t, true)

25
docs/roadmap/implementation.md
View File

@@ -2,8 +2,8 @@
**Last updated:** 2026-09-24
**Status:** Ready for implementation; all stages below are pending. Planning only:
no implementation is claimed by this document.
**Status:** Implemented. All stages below are complete; completion notes record
the checks performed for each unit of work.
## Scope and Execution Instructions
@@ -823,6 +823,27 @@ competing current contract or implicit legacy compatibility promise.
## Stage 14 — Verify the Integrated Resilience Contract
**Completed 2026-09-24.** Audited the target acceptance criteria against the
assembled CLI, runner, validator, reporting, and output-session tests. Added
controlled-provider subprocess coverage for all-proposal and all-validation
failure across repeated modules, all reporting/debug selections, a repaired and
filtered proposal response followed by a later failure, a repaired response with
invalid validation decisions, and signal cancellation with failed reports and no
transcript. Added an atomic-replacement cleanup check for a failed rename and
asserted ordinary reports omit transcript text and the configured secret. Existing
runner/validator tests cover survivor re-batching, fixed chunk membership and
cross-chunk rejection after text growth, segment metadata and ordering, and
active/queued cancellation; output-session tests cover initial
and final write failures, intentional replacement, and destination alias checks.
**Final validation:** `go test ./...` and `go vet ./...` passed. Focused
`go test -race` passed for runner, LLM, validators, process-report assembly,
reporting, diagnostics/output, and the subprocess package. The release build
script produced valid Linux amd64 and arm64 binaries using `v0.0.0` in a temporary
directory; a separate macOS arm64 cross-build also passed. No live provider or
released artifact was used; provider behavior is covered by deterministic local
HTTP fixtures. Builds were cross-compiled and not executed on target platforms.
**Work:** Audit every acceptance criterion in `resilience.md` against final code and
retained tests. Add only missing integration coverage. In particular verify the
following cross-component outcomes rather than relying solely on green leaf tests:

26
docs/roadmap/resilience.md
View File

@@ -2,9 +2,9 @@
**Last updated:** 2026-09-24
**Status:** Target behavior agreed; not implemented. This document defines the
desired end state and user intent. Concrete design decisions and implementation
steps are recorded in [implementation.md](implementation.md).
**Status:** Implemented. This document records the selected behavior and user
intent. Concrete design decisions and completion evidence are in
[implementation.md](implementation.md).
## Purpose and User Intent
@@ -33,20 +33,21 @@ Internal structures and measurements must serve Audita's processing workflow.
The reporting package adapts those outcomes to structured and human-facing
presentations, rather than reshaping processing around a reporting convenience.
## Current Baseline and Intended Change
## Historical Baseline and Implemented Change
The [pipeline](../internal/pipeline.md), [LLM integration](../integrations/openai-compatible-llm.md),
[report assembly](../internal/diagnostics-reporting.md), and
[artifact lifecycle](../operations.md) describe current behavior.
Currently, exhausted structural repairs can become warnings and skipped proposals
or rejected validation batches. Other model runtime failures cancel peer work,
stop subsequent modules, and prevent final transcript output. Successful repairs
appear in execution observations rather than a dedicated recovery classification.
Detailed artifacts are written during ordinary runs and may subsequently be
removed by retention. The working transcript is re-chunked between modules.
Before this work, exhausted structural repairs could become warnings and skipped
proposals or rejected validation batches. Other model runtime failures canceled
peer work, stopped subsequent modules, and prevented final transcript output.
Successful repairs appeared in execution observations rather than a dedicated
recovery classification. Detailed artifacts were written during ordinary runs
and could later be removed by retention. The working transcript was re-chunked
between modules.
This feature deliberately changes those behaviors. It builds on the completed
This feature changed those behaviors. It builds on the completed
[library adoption](library-adoption.md) and [review remediation](remediation.md).
Fatal cancellation remains necessary; unsuccessful model work must be separated
from failures that actually require cancellation of the run.
@@ -278,8 +279,7 @@ volume, and a short reason breakdown. Detailed inspection belongs in YAML or deb
failures, and reporting/debug selections. Preserve domain safeguards, capacity
limits, and redaction.
- Canonical CLI, configuration, operations, integration, internal, and architecture
documentation describes the implemented final behavior. Until implementation,
this roadmap does not change their current implemented contracts.
documentation describes the implemented final behavior.
## Implementation Reference

20
internal/core/diagnostics/output_session_test.go
View File

@@ -83,6 +83,26 @@ func TestPrepareOutputPreservesExistingTranscriptAndChecksParent(t *testing.T) {
}
}
func TestAtomicReplacementCleansTemporaryFileAfterRenameFailure(t *testing.T) {
dir := t.TempDir()
blocked := filepath.Join(dir, "directory-target")
if err := os.Mkdir(blocked, 0700); err != nil {
t.Fatal(err)
}
if err := atomicReplace(blocked, []byte("new content")); err == nil {
t.Fatal("directory replacement unexpectedly succeeded")
}
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if strings.HasPrefix(entry.Name(), ".audita-write-") {
t.Fatalf("replacement left temporary file: %s", entry.Name())
}
}
}
func TestDisabledReportingIgnoresUnusableDirectory(t *testing.T) {
dir := t.TempDir()
session, err := PrepareOutput(OutputSelection{TranscriptPath: filepath.Join(dir, "out.json"), ReportDir: filepath.Join(dir, "missing"), ReportingEnabled: false})

16
internal/framework/runner/runner_test.go
View File

@@ -279,7 +279,10 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
cfg.Chunking.MaxSectionTokens = 3
cfg.Chunking.MinSectionTokens = 0
transcript := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Text: "a"}, {ID: 2, Text: "b"}, {ID: 3, Text: "c"}, {ID: 4, Text: "d"},
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "a"},
{ID: 2, Speaker: "B", Start: 1, End: 2, Text: "b"},
{ID: 3, Speaker: "A", Start: 2, End: 3, Text: "c"},
{ID: 4, Speaker: "B", Start: 3, End: 4, Text: "d"},
}}
var mu sync.Mutex
seen := make(map[int][]int)
@@ -302,6 +305,9 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
if req.Section.Index == 0 && req.WorkingTranscript.Segments[0].Text != updatedText {
return nil, errors.New("later module did not see updated section text")
}
if req.Section.Index == 0 {
return []proposals.CorrectionProposal{{TargetSegmentID: 3, OriginalText: "c", CorrectedText: "C", Confidence: 1}}, nil
}
return nil, nil
}},
}})
@@ -315,9 +321,15 @@ func TestRunnerKeepsSectionMembershipAfterTextGrowth(t *testing.T) {
if len(seen) != 2 || len(seen[0]) != 2 || seen[0][0] != 1 || seen[0][1] != 2 || len(seen[1]) != 2 || seen[1][0] != 3 || seen[1][1] != 4 {
t.Fatalf("later module saw changed section membership: %+v", seen)
}
if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[3].ID != 4 {
if len(out.FinalTranscript.Segments) != 4 || out.FinalTranscript.Segments[0].Text != updatedText || out.FinalTranscript.Segments[2].Text != "c" || out.FinalTranscript.Segments[3].ID != 4 {
t.Fatalf("recombined transcript: %+v", out.FinalTranscript)
}
for i, segment := range out.FinalTranscript.Segments {
original := transcript.Segments[i]
if segment.ID != original.ID || segment.Speaker != original.Speaker || segment.Start != original.Start || segment.End != original.End {
t.Fatalf("segment %d metadata changed: %+v", i, segment)
}
}
if out.WorkUnits[0].CorrectionsApplied != 1 || out.WorkUnits[1].CorrectionsApplied != 0 {
t.Fatalf("section application counts: %+v", out.WorkUnits)
}