From 7cefb3fb04f09394881565e921ace97e8abbbae6 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 24 Sep 2026 02:40:48 +0000 Subject: [PATCH] Verify the integrated application and complete the migration --- cmd/audita/main_integration_test.go | 278 ++++++++++++++++++ docs/integrations/output-files.md | 2 +- .../integrations/transcript-glossary-files.md | 2 +- docs/internal/diagnostics-reporting.md | 1 - docs/internal/modules.md | 2 +- docs/internal/output-schemas.md | 2 +- docs/internal/pipeline.md | 1 - docs/internal/validators.md | 2 +- docs/operations.md | 8 +- docs/roadmap/configkit.md | 17 +- docs/roadmap/implementation.md | 18 +- docs/roadmap/library-adoption.md | 7 +- 12 files changed, 314 insertions(+), 26 deletions(-) create mode 100644 cmd/audita/main_integration_test.go diff --git a/cmd/audita/main_integration_test.go b/cmd/audita/main_integration_test.go new file mode 100644 index 0000000..1141592 --- /dev/null +++ b/cmd/audita/main_integration_test.go @@ -0,0 +1,278 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/cli" + "gitea.maximumdirect.net/eric/audita/internal/core/reporting" +) + +func TestProcessSubprocessHelper(t *testing.T) { + if os.Getenv("AUDITA_HELPER_PROCESS") != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Exit(cli.Run(os.Args[i+1:], os.Stdout, os.Stderr)) + } + } + os.Exit(2) +} + +type outboundCall struct { + Model string + Validation bool +} + +type controlledProvider struct { + mu sync.Mutex + calls []outboundCall + proposalCount int + mode string + secret string +} + +func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) { + var request struct { + Model string `json:"model"` + ResponseFormat struct { + JSONSchema struct { + Schema struct { + Properties map[string]any `json:"properties"` + } `json:"schema"` + } `json:"json_schema"` + } `json:"response_format"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, err.Error(), 400) + return + } + _, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"] + p.mu.Lock() + p.calls = append(p.calls, outboundCall{request.Model, validation}) + if !validation { + p.proposalCount++ + } + proposalCount := p.proposalCount + p.mu.Unlock() + if p.mode == "failure" { + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret) + return + } + response := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}` + if validation { + response = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}` + } + if p.mode == "repair" && !validation && proposalCount == 1 { + response = `{"wrong":true}` + } + if p.mode == "warning" && !validation { + response = `{"wrong":true}` + } + if p.mode == "noop" && !validation { + response = `{"corrections":[]}` + } + if p.mode == "low-confidence" && !validation { + response = `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.1}]}` + } + content, _ := json.Marshal(response) + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"model":"synthetic-result","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, content) +} + +type processFixture struct { + dir, config, transcript, glossary, report, output string + provider *controlledProvider + server *httptest.Server +} + +func newProcessFixture(t *testing.T, mode, modules string) processFixture { + t.Helper() + dir := t.TempDir() + p := &controlledProvider{mode: mode, secret: "synthetic-provider-secret-2468"} + server := httptest.NewServer(http.HandlerFunc(p.serve)) + t.Cleanup(server.Close) + if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil { + t.Fatal(err) + } + for name, model := range map[string]string{"proposal": "base-proposal", "validation": "base-validation", "proposal-testing": "testing-proposal", "validation-testing": "testing-validation", "alternate": "alternate-validation"} { + profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: %s\ntimeout_seconds: 5\n", name, model) + if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: proposal-testing\n validation_profile: validation-testing\n"), 0600); err != nil { + t.Fatal(err) + } + configPath := filepath.Join(dir, "config.yml") + configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 1\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n retention: always\n", server.URL, modules, filepath.Join(dir, "runs")) + if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil { + t.Fatal(err) + } + transcriptPath := filepath.Join(dir, "transcript.json") + if err := os.WriteFile(transcriptPath, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil { + t.Fatal(err) + } + glossaryPath := filepath.Join(dir, "glossary.yaml") + if err := os.WriteFile(glossaryPath, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil { + t.Fatal(err) + } + return processFixture{dir: dir, config: configPath, transcript: transcriptPath, glossary: glossaryPath, report: filepath.Join(dir, "report.json"), output: filepath.Join(dir, "output.json"), provider: p, server: server} +} + +func (f processFixture) run(t *testing.T, extra ...string) (int, string, string, reporting.ProcessReport) { + t.Helper() + args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--report-json", f.report, "--output", f.output} + args = append(args, extra...) + cmd := exec.Command(os.Args[0], args...) + cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret) + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + err := cmd.Run() + code := 0 + if err != nil { + if exit, ok := err.(*exec.ExitError); ok { + code = exit.ExitCode() + } else { + t.Fatal(err) + } + } + raw, err := os.ReadFile(f.report) + if err != nil { + t.Fatalf("report missing: %v, stderr=%s", err, stderr.String()) + } + var report reporting.ProcessReport + if err := json.Unmarshal(raw, &report); err != nil { + t.Fatal(err) + } + return code, stdout.String(), stderr.String(), report +} + +func (f processFixture) calls() []outboundCall { + f.provider.mu.Lock() + defer f.provider.mu.Unlock() + return append([]outboundCall(nil), f.provider.calls...) +} + +func TestSubprocessSelectionAndRepair(t *testing.T) { + for _, tc := range []struct { + name string + extra []string + proposal, validation string + }{ + {"base", nil, "base-proposal", "base-validation"}, + {"overlay and explicit validation", []string{"--overlay", "testing", "--validation-profile", "alternate", "--grammar-confidence-threshold", "0.9"}, "testing-proposal", "alternate-validation"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newProcessFixture(t, "repair", "[grammar]") + code, stdout, stderr, report := f.run(t, tc.extra...) + if code != 0 || stdout != "" || stderr != "" || report.Status != "success" { + t.Fatalf("code=%d stdout=%q stderr=%q report=%+v", code, stdout, stderr, report) + } + raw, err := os.ReadFile(f.output) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(raw, []byte("Hello world")) { + t.Fatalf("correction missing: %s", raw) + } + calls := f.calls() + if len(calls) < 3 || calls[0].Model != tc.proposal || calls[1].Model != tc.proposal { + t.Fatalf("proposal models: %+v", calls) + } + for _, call := range calls[2:] { + if !call.Validation || call.Model != tc.validation { + t.Fatalf("validation model: %+v", calls) + } + } + if report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 || report.LLMSummary.ProviderAttempts != uint64(len(calls)) { + t.Fatalf("LLM summary: %+v", report.LLMSummary) + } + if report.Diagnostics == nil || report.Diagnostics.LLMExecutionPath == "" || report.Diagnostics.SelectedProfilesPath == "" { + t.Fatalf("paths: %+v", report.Diagnostics) + } + }) + } +} + +func TestSubprocessWarningEmptyRepeatedAndFailure(t *testing.T) { + t.Run("domain rejection", func(t *testing.T) { + f := newProcessFixture(t, "low-confidence", "[grammar]") + code, _, stderr, report := f.run(t) + if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].ValidatorRejected) == 0 { + t.Fatalf("domain rejection: %d %q %+v", code, stderr, report) + } + if len(f.calls()) != 1 { + t.Fatalf("semantic rejection retried: %+v", f.calls()) + } + }) + t.Run("warning", func(t *testing.T) { + f := newProcessFixture(t, "warning", "[grammar]") + code, _, stderr, report := f.run(t) + if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) == 0 { + t.Fatalf("warning result: %d %q %+v", code, stderr, report) + } + if len(f.calls()) != 2 || report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 { + t.Fatalf("warning observations: %+v %+v", f.calls(), report.LLMSummary) + } + }) + t.Run("empty", func(t *testing.T) { + f := newProcessFixture(t, "noop", "[]") + code, _, stderr, report := f.run(t) + if code != 0 || stderr != "" || len(f.calls()) != 0 || report.LLMSummary == nil || !report.LLMSummary.Usage.Total.Known || !report.LLMSummary.Usage.Total.Complete { + t.Fatalf("empty result: %d %q %+v", code, stderr, report.LLMSummary) + } + }) + t.Run("repeated", func(t *testing.T) { + f := newProcessFixture(t, "noop", "[grammar, grammar]") + code, _, stderr, report := f.run(t) + if code != 0 || stderr != "" || len(report.ModuleResults) != 2 || report.ModuleResults[0].ModuleInstance == report.ModuleResults[1].ModuleInstance || len(f.calls()) != 2 { + t.Fatalf("repeated result: %d %q %+v %+v", code, stderr, report.ModuleResults, f.calls()) + } + }) + t.Run("failure redaction", func(t *testing.T) { + f := newProcessFixture(t, "failure", "[grammar]") + code, stdout, stderr, report := f.run(t) + if code != 1 || stdout != "" || report.Status != "failed" || report.ErrorPhase != "runner_execution" || report.LLMSummary == nil || report.LLMSummary.LogicalCalls != 1 { + t.Fatalf("failure result: %d %q %q %+v", code, stdout, stderr, report) + } + if strings.Contains(stderr, f.provider.secret) { + t.Fatal("secret in stderr") + } + for _, path := range []string{f.report, report.Diagnostics.DirectoryPath} { + _ = filepath.Walk(path, func(name string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + if info.IsDir() { + return nil + } + content, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(content, []byte(f.provider.secret)) { + t.Fatalf("secret in %s", name) + } + return nil + }) + } + if _, err := os.Stat(report.Diagnostics.LLMExecutionPath); err != nil { + t.Fatal(err) + } + }) +} diff --git a/docs/integrations/output-files.md b/docs/integrations/output-files.md index e1272ef..ac50c11 100644 --- a/docs/integrations/output-files.md +++ b/docs/integrations/output-files.md @@ -9,7 +9,7 @@ the [CLI](../cli.md#streams-and-exit-status). ## Corrected Transcript -The [output setting](../config.md#pipeline-and-output) selects the envelope: +The [output setting](../config.md#settings) selects the envelope: | Encoder | JSON shape | | --- | --- | diff --git a/docs/integrations/transcript-glossary-files.md b/docs/integrations/transcript-glossary-files.md index 35d1394..2af4be9 100644 --- a/docs/integrations/transcript-glossary-files.md +++ b/docs/integrations/transcript-glossary-files.md @@ -42,7 +42,7 @@ The top-level object contains a `glossary` list with at least one entry. The parser checks empty strings without trimming them. Unknown YAML fields are ignored by the glossary parser, unlike the stricter -[configuration-file schema](../config.md#file-schema). +[configuration-file schema](../config.md#loading-and-precedence). The [tiny glossary](../../examples/tiny-glossary.yaml) is a complete copyable input. Malformed YAML, an empty/missing glossary, invalid shapes, missing required text, diff --git a/docs/internal/diagnostics-reporting.md b/docs/internal/diagnostics-reporting.md index 1458c23..ededbff 100644 --- a/docs/internal/diagnostics-reporting.md +++ b/docs/internal/diagnostics-reporting.md @@ -75,5 +75,4 @@ Operational handling of transcript-bearing artifacts belongs to - `internal/core/diagnostics/*_test.go` - `internal/framework/processreport/*_test.go` - `internal/core/reporting/report_test.go` -- `internal/cli/run_test.go` - `cmd/audita/main_integration_test.go` diff --git a/docs/internal/modules.md b/docs/internal/modules.md index e27148d..9a417a6 100644 --- a/docs/internal/modules.md +++ b/docs/internal/modules.md @@ -3,7 +3,7 @@ ## Scope This document covers internal module contracts and packages. User-selectable keys -and the default sequence belong to [configuration](../config.md#pipeline-and-output). +and the default sequence belong to [configuration](../config.md#settings). ## Module Contract diff --git a/docs/internal/output-schemas.md b/docs/internal/output-schemas.md index b4d2407..9cfa7b8 100644 --- a/docs/internal/output-schemas.md +++ b/docs/internal/output-schemas.md @@ -6,7 +6,7 @@ JSON envelopes and segment fields are defined by the [output-file contract](../integrations/output-files.md#corrected-transcript); selectable values and defaults belong to -[configuration](../config.md#pipeline-and-output). +[configuration](../config.md#settings). ## Resolution and Encoding diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 9f27e06..4a6e3ee 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -63,5 +63,4 @@ reports are derived from actual run results. Global requirements are owned by th - `internal/framework/runner/runner_test.go` - `internal/framework/proposal_generation/*_test.go` -- `internal/cli/run_test.go` - `cmd/audita/main_integration_test.go` diff --git a/docs/internal/validators.md b/docs/internal/validators.md index 982682b..78f3db4 100644 --- a/docs/internal/validators.md +++ b/docs/internal/validators.md @@ -4,7 +4,7 @@ This document describes internal validator composition, execution order, and decision handling. These registry keys identify implementation components; they -are not independently selectable through [configuration](../config.md#pipeline-and-output). +are not independently selectable through [configuration](../config.md#settings). ## Ownership diff --git a/docs/operations.md b/docs/operations.md index e00aafc..166da31 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -37,7 +37,7 @@ output files elsewhere. ## Diagnostics Directory -The [diagnostics configuration](config.md#thresholds-context-and-diagnostics) +The [diagnostics configuration](config.md#settings) selects the parent work directory. Runs create children named like `run-`. The runtime creates the work directory if needed. Report paths may be relative when the supplied work directory is relative; @@ -68,7 +68,7 @@ write succeeded or that the files still exist after retention. ## Retention Behavior -[Configuration](config.md#thresholds-context-and-diagnostics) owns accepted +[Configuration](config.md#settings) owns accepted retention values and the default. Current lifecycle behavior is: | Run outcome / selection | Directory outcome | @@ -89,10 +89,10 @@ Run artifacts can contain full transcripts, glossary-related context, and LLM prompts/responses. Configured-key redaction does not anonymize those contents. Choose a work directory whose ownership and permissions fit the transcript data, and retain or share artifacts intentionally. Credential-supply mechanisms are -owned by [configuration](config.md#secrets). +owned by [configuration](config.md#credentials). For a debugging run, select retained diagnostics using the -[retention flag](cli.md#process-flag-reference). Copy needed artifacts before +[retention flag](cli.md#setting-overrides). Copy needed artifacts before removing completed run directories manually. Avoid deleting a directory while its run is active. Audita does not provide a separate archival/backup or remote artifact storage facility. diff --git a/docs/roadmap/configkit.md b/docs/roadmap/configkit.md index 49b19d4..f929e54 100644 --- a/docs/roadmap/configkit.md +++ b/docs/roadmap/configkit.md @@ -2,9 +2,9 @@ **Last updated:** 2026-09-24 -**Status:** Final target selected; production implementation pending. The boundary -and selected overlays are verified. The [implementation plan](implementation.md) -owns execution stages; no production migration or release has begun. +**Status:** Production implementation complete. The selected configuration and +overlay model is implemented; [implementation evidence](implementation.md) +records verification. No release has been published. ## Purpose and Direction @@ -20,15 +20,14 @@ guide instead of legacy loaders, aliases, dual operating modes, or an ongoing compatibility layer. Retain existing syntax only when it independently suits the new model, not because compatibility requires it. -The initial review and prototype used ConfigKit v0.3.1. The prototype now pins -v0.4.0 and verifies its selected-overlay path; original v0.3.1 findings remain -historical. The installed -application must depend on a versioned module, not a sibling checkout or local -module replacement. +The initial review and prototype used ConfigKit v0.3.1, then verified v0.4.0 +selected overlays. Those findings remain historical; the prototype was removed +after production integration. The application depends on a versioned module, +without a sibling checkout or local module replacement. The shared boundary/prototype unit is complete. Its [selected contract, inventory, and evidence](library-boundary-findings.md) define the detailed target alongside -this roadmap. Production configuration migration has not begun. +this roadmap. Production configuration migration is complete. ## Target End State diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f79a8f4..bc43b5a 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -2,7 +2,7 @@ **Last updated:** 2026-09-24 -**Status:** Ready for implementation. All stages below are pending. This replaces +**Status:** Implemented. All twelve stages are complete. This replaced the completed testing-overlay prototype plan; its results remain in the [boundary findings](library-boundary-findings.md#selected-overlay-extension-2026-09-24). @@ -583,7 +583,21 @@ belongs to Stage 12. ## Stage 12: Verify the Combined Application and Close the Refactor -**Status:** Pending. **Depends on:** Stages 1–11. +**Status:** Complete. Controlled HTTP subprocess tests passed for base and testing +overlay selection, explicit validation-profile and domain overrides, both outbound +models, accepted repair, domain rejection without semantic retry, warning-only +repair exhaustion, empty and repeated module pipelines, and safe partial failure +diagnostics. `go test ./...`, `go build -o ./bin/audita ./cmd/audita`, focused +race tests for runtime/runner/subprocess code, and both Linux release builds +passed. The full suite also passed under Go 1.25.5 with offline module access. +Maintained examples and local documentation links validated; `git diff --check` +passed. No native macOS or live provider call was run. Release preparation remains +separate: propose Audita v2.0.0 with [one migration guide](../migrations/v2.md), +the actual pinned production dependency/license inventory, and LGPL +notice/source/rebuild-distribution review before publication. No release was +published here. + +**Depends on:** Stages 1–11. **Owns:** final integration tests, final defect fixes within this scope, and status/evidence. 1. Complete representative production subprocess workflows against controlled HTTP: diff --git a/docs/roadmap/library-adoption.md b/docs/roadmap/library-adoption.md index 7136c13..a653250 100644 --- a/docs/roadmap/library-adoption.md +++ b/docs/roadmap/library-adoption.md @@ -2,10 +2,9 @@ **Last updated:** 2026-09-24 -**Status:** Final target selected; production implementation pending. The shared -boundary and offline prototype, including ConfigKit v0.4.0 overlays, are verified. -[Implementation stages](implementation.md) are owned by the implementation plan. -This roadmap defines the end state and user intent; it authorizes no publication. +**Status:** Production implementation complete. ConfigKit v0.4.0 and PromptKit +v0.11.0 are adopted; [implementation evidence](implementation.md) records the +completed checks. Release compliance and publication remain separate work. ## Goal and Ownership