Verify the integrated application and complete the migration
This commit is contained in:
278
cmd/audita/main_integration_test.go
Normal file
278
cmd/audita/main_integration_test.go
Normal file
@@ -0,0 +1,278 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
)
|
||||
|
||||
func TestProcessSubprocessHelper(t *testing.T) {
|
||||
if os.Getenv("AUDITA_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
for i, arg := range os.Args {
|
||||
if arg == "--" {
|
||||
os.Exit(cli.Run(os.Args[i+1:], os.Stdout, os.Stderr))
|
||||
}
|
||||
}
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
type outboundCall struct {
|
||||
Model string
|
||||
Validation bool
|
||||
}
|
||||
|
||||
type controlledProvider struct {
|
||||
mu sync.Mutex
|
||||
calls []outboundCall
|
||||
proposalCount int
|
||||
mode string
|
||||
secret string
|
||||
}
|
||||
|
||||
func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
ResponseFormat struct {
|
||||
JSONSchema struct {
|
||||
Schema struct {
|
||||
Properties map[string]any `json:"properties"`
|
||||
} `json:"schema"`
|
||||
} `json:"json_schema"`
|
||||
} `json:"response_format"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
||||
p.mu.Lock()
|
||||
p.calls = append(p.calls, outboundCall{request.Model, validation})
|
||||
if !validation {
|
||||
p.proposalCount++
|
||||
}
|
||||
proposalCount := p.proposalCount
|
||||
p.mu.Unlock()
|
||||
if p.mode == "failure" {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
|
||||
return
|
||||
}
|
||||
response := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
if validation {
|
||||
response = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
}
|
||||
if p.mode == "repair" && !validation && proposalCount == 1 {
|
||||
response = `{"wrong":true}`
|
||||
}
|
||||
if p.mode == "warning" && !validation {
|
||||
response = `{"wrong":true}`
|
||||
}
|
||||
if p.mode == "noop" && !validation {
|
||||
response = `{"corrections":[]}`
|
||||
}
|
||||
if p.mode == "low-confidence" && !validation {
|
||||
response = `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.1}]}`
|
||||
}
|
||||
content, _ := json.Marshal(response)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic-result","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, content)
|
||||
}
|
||||
|
||||
type processFixture struct {
|
||||
dir, config, transcript, glossary, report, output string
|
||||
provider *controlledProvider
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
p := &controlledProvider{mode: mode, secret: "synthetic-provider-secret-2468"}
|
||||
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
||||
t.Cleanup(server.Close)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, model := range map[string]string{"proposal": "base-proposal", "validation": "base-validation", "proposal-testing": "testing-proposal", "validation-testing": "testing-validation", "alternate": "alternate-validation"} {
|
||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: %s\ntimeout_seconds: 5\n", name, model)
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: proposal-testing\n validation_profile: validation-testing\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 1\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n retention: always\n", server.URL, modules, filepath.Join(dir, "runs"))
|
||||
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
glossaryPath := filepath.Join(dir, "glossary.yaml")
|
||||
if err := os.WriteFile(glossaryPath, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return processFixture{dir: dir, config: configPath, transcript: transcriptPath, glossary: glossaryPath, report: filepath.Join(dir, "report.json"), output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
||||
}
|
||||
|
||||
func (f processFixture) run(t *testing.T, extra ...string) (int, string, string, reporting.ProcessReport) {
|
||||
t.Helper()
|
||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--report-json", f.report, "--output", f.output}
|
||||
args = append(args, extra...)
|
||||
cmd := exec.Command(os.Args[0], args...)
|
||||
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
code := 0
|
||||
if err != nil {
|
||||
if exit, ok := err.(*exec.ExitError); ok {
|
||||
code = exit.ExitCode()
|
||||
} else {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(f.report)
|
||||
if err != nil {
|
||||
t.Fatalf("report missing: %v, stderr=%s", err, stderr.String())
|
||||
}
|
||||
var report reporting.ProcessReport
|
||||
if err := json.Unmarshal(raw, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return code, stdout.String(), stderr.String(), report
|
||||
}
|
||||
|
||||
func (f processFixture) calls() []outboundCall {
|
||||
f.provider.mu.Lock()
|
||||
defer f.provider.mu.Unlock()
|
||||
return append([]outboundCall(nil), f.provider.calls...)
|
||||
}
|
||||
|
||||
func TestSubprocessSelectionAndRepair(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
extra []string
|
||||
proposal, validation string
|
||||
}{
|
||||
{"base", nil, "base-proposal", "base-validation"},
|
||||
{"overlay and explicit validation", []string{"--overlay", "testing", "--validation-profile", "alternate", "--grammar-confidence-threshold", "0.9"}, "testing-proposal", "alternate-validation"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newProcessFixture(t, "repair", "[grammar]")
|
||||
code, stdout, stderr, report := f.run(t, tc.extra...)
|
||||
if code != 0 || stdout != "" || stderr != "" || report.Status != "success" {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q report=%+v", code, stdout, stderr, report)
|
||||
}
|
||||
raw, err := os.ReadFile(f.output)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(raw, []byte("Hello world")) {
|
||||
t.Fatalf("correction missing: %s", raw)
|
||||
}
|
||||
calls := f.calls()
|
||||
if len(calls) < 3 || calls[0].Model != tc.proposal || calls[1].Model != tc.proposal {
|
||||
t.Fatalf("proposal models: %+v", calls)
|
||||
}
|
||||
for _, call := range calls[2:] {
|
||||
if !call.Validation || call.Model != tc.validation {
|
||||
t.Fatalf("validation model: %+v", calls)
|
||||
}
|
||||
}
|
||||
if report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 || report.LLMSummary.ProviderAttempts != uint64(len(calls)) {
|
||||
t.Fatalf("LLM summary: %+v", report.LLMSummary)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.LLMExecutionPath == "" || report.Diagnostics.SelectedProfilesPath == "" {
|
||||
t.Fatalf("paths: %+v", report.Diagnostics)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessWarningEmptyRepeatedAndFailure(t *testing.T) {
|
||||
t.Run("domain rejection", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "low-confidence", "[grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].ValidatorRejected) == 0 {
|
||||
t.Fatalf("domain rejection: %d %q %+v", code, stderr, report)
|
||||
}
|
||||
if len(f.calls()) != 1 {
|
||||
t.Fatalf("semantic rejection retried: %+v", f.calls())
|
||||
}
|
||||
})
|
||||
t.Run("warning", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "warning", "[grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) == 0 {
|
||||
t.Fatalf("warning result: %d %q %+v", code, stderr, report)
|
||||
}
|
||||
if len(f.calls()) != 2 || report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 {
|
||||
t.Fatalf("warning observations: %+v %+v", f.calls(), report.LLMSummary)
|
||||
}
|
||||
})
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "noop", "[]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || len(f.calls()) != 0 || report.LLMSummary == nil || !report.LLMSummary.Usage.Total.Known || !report.LLMSummary.Usage.Total.Complete {
|
||||
t.Fatalf("empty result: %d %q %+v", code, stderr, report.LLMSummary)
|
||||
}
|
||||
})
|
||||
t.Run("repeated", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "noop", "[grammar, grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || len(report.ModuleResults) != 2 || report.ModuleResults[0].ModuleInstance == report.ModuleResults[1].ModuleInstance || len(f.calls()) != 2 {
|
||||
t.Fatalf("repeated result: %d %q %+v %+v", code, stderr, report.ModuleResults, f.calls())
|
||||
}
|
||||
})
|
||||
t.Run("failure redaction", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "failure", "[grammar]")
|
||||
code, stdout, stderr, report := f.run(t)
|
||||
if code != 1 || stdout != "" || report.Status != "failed" || report.ErrorPhase != "runner_execution" || report.LLMSummary == nil || report.LLMSummary.LogicalCalls != 1 {
|
||||
t.Fatalf("failure result: %d %q %q %+v", code, stdout, stderr, report)
|
||||
}
|
||||
if strings.Contains(stderr, f.provider.secret) {
|
||||
t.Fatal("secret in stderr")
|
||||
}
|
||||
for _, path := range []string{f.report, report.Diagnostics.DirectoryPath} {
|
||||
_ = filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
content, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(content, []byte(f.provider.secret)) {
|
||||
t.Fatalf("secret in %s", name)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if _, err := os.Stat(report.Diagnostics.LLMExecutionPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user