eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Fix the five library adoption review findings

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 15:32:13 +00:00
parent c42bc01052
commit 665ee85215
19 changed files with 942 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
cmd/audita/main_integration_test.go
View File

@@ -35,16 +35,20 @@ type outboundCall struct {
}
type controlledProvider struct {
mu sync.Mutex
calls []outboundCall
proposalCount int
mode string
secret string
mu sync.Mutex
calls []outboundCall
proposalCount int
mode string
secret string
unselectedSecret string
}
func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
var request struct {
Model string `json:"model"`
Model string `json:"model"`
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
ResponseFormat struct {
JSONSchema struct {
Schema struct {
@@ -65,7 +69,7 @@ func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
}
proposalCount := p.proposalCount
p.mu.Unlock()
if p.mode == "failure" {
if p.mode == "failure" || (p.mode == "secret-partial-failure" && !validation && proposalCount > 1) {
w.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
return
@@ -86,6 +90,53 @@ func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
if p.mode == "low-confidence" && !validation {
response = `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.1}]}`
}
if validation && strings.HasPrefix(p.mode, "secret-") {
payload := map[string]any{"validations": []any{map[string]any{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "echo:" + p.secret + " unused:" + p.unselectedSecret}}}
raw, _ := json.Marshal(payload)
response = string(raw)
}
if p.mode == "sections" {
for _, message := range request.Messages {
marker := "Transcript section:"
if validation {
marker = "Corrections to validate:"
}
_, payload, found := strings.Cut(message.Content, marker)
if !found {
continue
}
var items []map[string]any
var parseErr error
if validation {
parseErr = json.Unmarshal([]byte(strings.TrimSpace(payload)), &items)
} else {
var section struct {
Segments []map[string]any `json:"segments"`
}
parseErr = json.Unmarshal([]byte(strings.TrimSpace(payload)), &section)
items = section.Segments
}
if parseErr != nil {
http.Error(w, parseErr.Error(), 500)
return
}
results := []map[string]any{}
for _, item := range items {
if validation {
results = append(results, map[string]any{"correction_index": item["correction_index"], "approved": false, "confidence": 0.99, "reason": "test rejection"})
} else {
results = append(results, map[string]any{"id": item["id"], "original_text": item["text"], "corrected_text": strings.ToUpper(item["text"].(string)), "confidence": 0.99})
}
}
key := "corrections"
if validation {
key = "validations"
}
raw, _ := json.Marshal(map[string]any{key: results})
response = string(raw)
}
}
content, _ := json.Marshal(response)
w.Header().Set("Content-Type", "application/json")
_, _ = fmt.Fprintf(w, `{"model":"synthetic-result","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, content)
@@ -139,7 +190,7 @@ func (f processFixture) run(t *testing.T, extra ...string) (int, string, string,
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--report-json", f.report, "--output", f.output}
args = append(args, extra...)
cmd := exec.Command(os.Args[0], args...)
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret, "AUDITA_TEST_UNUSED_KEY="+f.provider.unselectedSecret)
var stdout, stderr bytes.Buffer
cmd.Stdout, cmd.Stderr = &stdout, &stderr
err := cmd.Run()
@@ -276,3 +327,203 @@ func TestSubprocessWarningEmptyRepeatedAndFailure(t *testing.T) {
}
})
}
func TestSubprocessRedactsValidModelReasonsOnSuccessAndPartialFailure(t *testing.T) {
for _, mode := range []string{"secret-rejection", "secret-partial-failure"} {
t.Run(mode, func(t *testing.T) {
f := newProcessFixture(t, mode, "[grammar, grammar]")
f.provider.secret = `selected-credential-"\-123`
f.provider.unselectedSecret = `unused-credential-"\-456`
raw, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
raw = []byte(strings.Replace(string(raw), "backends:\n", "backends:\n unused:\n endpoint: "+f.server.URL+"/v1\n credential: AUDITA_TEST_UNUSED_KEY\n", 1))
if err := os.WriteFile(f.config, raw, 0600); err != nil {
t.Fatal(err)
}
code, _, stderr, report := f.run(t)
if mode == "secret-rejection" && (code != 0 || report.Status != "success") {
t.Fatalf("code=%d %s", code, stderr)
}
if mode == "secret-partial-failure" {
if code == 0 || report.Status != "failed" {
t.Fatalf("expected failure: %d %+v", code, report)
}
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
t.Fatalf("failure wrote complete output: %v", err)
}
}
if len(report.ModuleResults) == 0 || len(report.ModuleResults[0].ValidatorRejected) == 0 {
t.Fatalf("missing rejected domain result: %+v", report)
}
reason := report.ModuleResults[0].ValidatorRejected[0].Message
if !strings.Contains(reason, "[REDACTED]") {
t.Fatalf("reason not exercised: %s", reason)
}
sawLedger := false
sawInteraction := false
check := func(path string) {
t.Helper()
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var walk func(any)
walk = func(value any) {
switch v := value.(type) {
case string:
for _, secret := range []string{f.provider.secret, f.provider.unselectedSecret} {
escaped, _ := json.Marshal(secret)
if strings.Contains(v, secret) || strings.Contains(v, string(escaped[1:len(escaped)-1])) {
t.Fatalf("credential in %s", path)
}
}
case []any:
for _, item := range v {
walk(item)
}
case map[string]any:
for key, item := range v {
walk(key)
walk(item)
}
}
}
if strings.HasSuffix(path, ".json") {
var value any
if err := json.Unmarshal(raw, &value); err != nil {
t.Fatal(err)
}
walk(value)
} else {
walk(string(raw))
}
if strings.HasSuffix(path, "correction-ledger.json") {
sawLedger = true
if !strings.Contains(string(raw), "[REDACTED]") {
t.Fatal("ledger reason not exercised")
}
}
if strings.HasSuffix(path, "response-payload.json") {
sawInteraction = true
}
}
check(f.report)
err = filepath.WalkDir(filepath.Join(f.dir, "runs"), func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() {
check(path)
}
return nil
})
if err != nil {
t.Fatal(err)
}
if !sawLedger || !sawInteraction {
t.Fatal("missing diagnostic artifacts")
}
})
}
}
func TestSubprocessRepeatedModulesKeepSectionAttribution(t *testing.T) {
f := newProcessFixture(t, "sections", "[grammar, grammar]")
raw, err := os.ReadFile(f.config)
if err != nil {
t.Fatal(err)
}
raw = append(raw, []byte("chunking:\n max_section_tokens: 3\n min_section_tokens: 1\n")...)
if err := os.WriteFile(f.config, raw, 0600); err != nil {
t.Fatal(err)
}
input := `[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"},{"id":2,"speaker":"B","start":2,"end":3,"text":"good morning"}]`
if err := os.WriteFile(f.transcript, []byte(input), 0600); err != nil {
t.Fatal(err)
}
code, _, stderr, report := f.run(t)
if code != 0 {
t.Fatalf("process: %d %s", code, stderr)
}
if len(report.ModuleResults) != 2 {
t.Fatalf("module results: %+v", report)
}
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "*"))
if err != nil || len(dirs) != 1 {
t.Fatalf("run directories: %v %v", dirs, err)
}
raw, err = os.ReadFile(filepath.Join(dirs[0], "utilization-diagnostics.json"))
if err != nil {
t.Fatal(err)
}
var utilization reporting.UtilizationDiagnostics
if err := json.Unmarshal(raw, &utilization); err != nil {
t.Fatal(err)
}
if len(utilization.Modules) != 2 {
t.Fatalf("spurious modules: %+v", utilization.Modules)
}
for i, module := range utilization.Modules {
if module.ModuleInstance != report.ModuleResults[i].ModuleInstance || module.ProposalLLMCalls != 2 || module.ValidationLLMCalls != 2 {
t.Fatalf("module attribution: %+v", utilization.Modules)
}
}
raw, err = os.ReadFile(filepath.Join(dirs[0], "llm-execution.json"))
if err != nil {
t.Fatal(err)
}
var decoded struct {
Calls []struct {
Context struct {
Role string `json:"role"`
ModuleInstance string `json:"module_instance"`
SectionIndex *int `json:"section_index"`
BatchIndex *int `json:"batch_index"`
} `json:"context"`
} `json:"calls"`
}
if err := json.Unmarshal(raw, &decoded); err != nil {
t.Fatal(err)
}
seen := map[string]map[int]bool{}
for _, call := range decoded.Calls {
c := call.Context
if c.Role != "validation" {
continue
}
if c.SectionIndex == nil || c.BatchIndex == nil || *c.BatchIndex != 0 {
t.Fatalf("missing section/batch: %+v", c)
}
if seen[c.ModuleInstance] == nil {
seen[c.ModuleInstance] = map[int]bool{}
}
seen[c.ModuleInstance][*c.SectionIndex] = true
}
for _, module := range utilization.Modules {
if len(seen[module.ModuleInstance]) != 2 {
t.Fatalf("section attribution: %+v", seen)
}
}
responseCount := 0
if err := filepath.WalkDir(dirs[0], func(path string, entry os.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() && strings.Contains(path, "section-") && strings.Contains(path, "batch-") && strings.HasSuffix(path, "response-payload.json") {
responseCount++
}
return nil
}); err != nil {
t.Fatal(err)
}
if responseCount != 4 {
t.Fatalf("validation artifact collision: %d", responseCount)
}
for _, module := range report.ModuleResults {
if len(module.ValidatorRejected) != 2 || module.ValidatorRejected[0].ProposalIndex != 0 || module.ValidatorRejected[1].ProposalIndex != 1 {
t.Fatalf("decision order: %+v", module.ValidatorRejected)
}
}
}