Compose resilient CLI processing and report lifecycle
This commit is contained in:
@@ -16,6 +16,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestProcessSubprocessHelper(t *testing.T) {
|
||||
@@ -30,26 +31,16 @@ func TestProcessSubprocessHelper(t *testing.T) {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
type outboundCall struct {
|
||||
Model string
|
||||
Validation bool
|
||||
type fixtureProvider struct {
|
||||
mu sync.Mutex
|
||||
calls int
|
||||
validationCalls int
|
||||
mode string
|
||||
secret string
|
||||
}
|
||||
|
||||
type controlledProvider struct {
|
||||
mu sync.Mutex
|
||||
calls []outboundCall
|
||||
proposalCount int
|
||||
mode string
|
||||
secret string
|
||||
unselectedSecret string
|
||||
}
|
||||
|
||||
func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
var request struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
ResponseFormat struct {
|
||||
JSONSchema struct {
|
||||
Schema struct {
|
||||
@@ -59,142 +50,87 @@ func (p *controlledProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
} `json:"response_format"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, "bad request", 400)
|
||||
return
|
||||
}
|
||||
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
||||
p.mu.Lock()
|
||||
p.calls = append(p.calls, outboundCall{request.Model, validation})
|
||||
if !validation {
|
||||
p.proposalCount++
|
||||
p.calls++
|
||||
if validation {
|
||||
p.validationCalls++
|
||||
}
|
||||
proposalCount := p.proposalCount
|
||||
validationCall := p.validationCalls
|
||||
p.mu.Unlock()
|
||||
if p.mode == "failure" || (p.mode == "secret-partial-failure" && !validation && proposalCount > 1) {
|
||||
if p.mode == "all-model-failure" || (p.mode == "partial-validation" && validation && validationCall == 1) {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
|
||||
return
|
||||
}
|
||||
response := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
if validation {
|
||||
response = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
}
|
||||
if p.mode == "repair" && !validation && proposalCount == 1 {
|
||||
response = `{"wrong":true}`
|
||||
}
|
||||
if p.mode == "warning" && !validation {
|
||||
response = `{"wrong":true}`
|
||||
}
|
||||
if p.mode == "noop" && !validation {
|
||||
response = `{"corrections":[]}`
|
||||
}
|
||||
if p.mode == "low-confidence" && !validation {
|
||||
response = `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.1}]}`
|
||||
}
|
||||
if validation && strings.HasPrefix(p.mode, "secret-") {
|
||||
payload := map[string]any{"validations": []any{map[string]any{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "echo:" + p.secret + " unused:" + p.unselectedSecret}}}
|
||||
raw, _ := json.Marshal(payload)
|
||||
response = string(raw)
|
||||
}
|
||||
if p.mode == "sections" {
|
||||
for _, message := range request.Messages {
|
||||
marker := "Transcript section:"
|
||||
if validation {
|
||||
marker = "Corrections to validate:"
|
||||
}
|
||||
_, payload, found := strings.Cut(message.Content, marker)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
var items []map[string]any
|
||||
var parseErr error
|
||||
if validation {
|
||||
parseErr = json.Unmarshal([]byte(strings.TrimSpace(payload)), &items)
|
||||
} else {
|
||||
var section struct {
|
||||
Segments []map[string]any `json:"segments"`
|
||||
}
|
||||
parseErr = json.Unmarshal([]byte(strings.TrimSpace(payload)), §ion)
|
||||
items = section.Segments
|
||||
}
|
||||
if parseErr != nil {
|
||||
http.Error(w, parseErr.Error(), 500)
|
||||
return
|
||||
}
|
||||
|
||||
results := []map[string]any{}
|
||||
for _, item := range items {
|
||||
if validation {
|
||||
results = append(results, map[string]any{"correction_index": item["correction_index"], "approved": false, "confidence": 0.99, "reason": "test rejection"})
|
||||
} else {
|
||||
results = append(results, map[string]any{"id": item["id"], "original_text": item["text"], "corrected_text": strings.ToUpper(item["text"].(string)), "confidence": 0.99})
|
||||
}
|
||||
}
|
||||
key := "corrections"
|
||||
if validation {
|
||||
key = "validations"
|
||||
}
|
||||
raw, _ := json.Marshal(map[string]any{key: results})
|
||||
response = string(raw)
|
||||
}
|
||||
}
|
||||
content, _ := json.Marshal(response)
|
||||
encoded, _ := json.Marshal(content)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic-result","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, content)
|
||||
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
||||
}
|
||||
|
||||
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
|
||||
|
||||
type processFixture struct {
|
||||
dir, config, transcript, glossary, report, output string
|
||||
provider *controlledProvider
|
||||
server *httptest.Server
|
||||
dir string
|
||||
config string
|
||||
transcript string
|
||||
glossary string
|
||||
output string
|
||||
provider *fixtureProvider
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
p := &controlledProvider{mode: mode, secret: "synthetic-provider-secret-2468"}
|
||||
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`}
|
||||
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
||||
t.Cleanup(server.Close)
|
||||
if err := os.MkdirAll(filepath.Join(dir, "profiles"), 0700); err != nil {
|
||||
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(dir, "overlays"), 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, model := range map[string]string{"proposal": "base-proposal", "validation": "base-validation", "proposal-testing": "testing-proposal", "validation-testing": "testing-validation", "alternate": "alternate-validation"} {
|
||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: %s\ntimeout_seconds: 5\n", name, model)
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", name+".yaml"), []byte(profile), 0600); err != nil {
|
||||
for _, id := range []string{"proposal", "validation"} {
|
||||
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n", id)
|
||||
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "overlays", "testing.yml"), []byte("llm:\n proposal_profile: proposal-testing\n validation_profile: validation-testing\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
configPath := filepath.Join(dir, "config.yml")
|
||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\n concurrency: 2\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 1\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n retention: always\n", server.URL, modules, filepath.Join(dir, "runs"))
|
||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: 0\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, modules, filepath.Join(dir, "runs"))
|
||||
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcriptPath := filepath.Join(dir, "transcript.json")
|
||||
if err := os.WriteFile(transcriptPath, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
||||
transcript := filepath.Join(dir, "input.json")
|
||||
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
glossaryPath := filepath.Join(dir, "glossary.yaml")
|
||||
if err := os.WriteFile(glossaryPath, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
||||
glossary := filepath.Join(dir, "glossary.yaml")
|
||||
if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return processFixture{dir: dir, config: configPath, transcript: transcriptPath, glossary: glossaryPath, report: filepath.Join(dir, "report.json"), output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
||||
return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary,
|
||||
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
||||
}
|
||||
|
||||
func (f processFixture) run(t *testing.T, extra ...string) (int, string, string, reporting.ProcessReport) {
|
||||
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
||||
t.Helper()
|
||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--report-json", f.report, "--output", f.output}
|
||||
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
|
||||
if includeOutput {
|
||||
args = append(args, "--output", f.output)
|
||||
}
|
||||
args = append(args, extra...)
|
||||
cmd := exec.Command(os.Args[0], args...)
|
||||
cmd.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret, "AUDITA_TEST_UNUSED_KEY="+f.provider.unselectedSecret)
|
||||
command := exec.Command(os.Args[0], args...)
|
||||
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
err := cmd.Run()
|
||||
command.Stdout, command.Stderr = &stdout, &stderr
|
||||
err := command.Run()
|
||||
code := 0
|
||||
if err != nil {
|
||||
if exit, ok := err.(*exec.ExitError); ok {
|
||||
@@ -203,328 +139,194 @@ func (f processFixture) run(t *testing.T, extra ...string) (int, string, string,
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
raw, err := os.ReadFile(f.report)
|
||||
return code, stdout.String(), stderr.String()
|
||||
}
|
||||
|
||||
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
||||
if err != nil {
|
||||
t.Fatalf("report missing: %v, stderr=%s", err, stderr.String())
|
||||
}
|
||||
var report reporting.ProcessReport
|
||||
if err := json.Unmarshal(raw, &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return code, stdout.String(), stderr.String(), report
|
||||
}
|
||||
|
||||
func (f processFixture) calls() []outboundCall {
|
||||
f.provider.mu.Lock()
|
||||
defer f.provider.mu.Unlock()
|
||||
return append([]outboundCall(nil), f.provider.calls...)
|
||||
}
|
||||
|
||||
func TestSubprocessSelectionAndRepair(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
extra []string
|
||||
proposal, validation string
|
||||
}{
|
||||
{"base", nil, "base-proposal", "base-validation"},
|
||||
{"overlay and explicit validation", []string{"--overlay", "testing", "--validation-profile", "alternate", "--grammar-confidence-threshold", "0.9"}, "testing-proposal", "alternate-validation"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
f := newProcessFixture(t, "repair", "[grammar]")
|
||||
code, stdout, stderr, report := f.run(t, tc.extra...)
|
||||
if code != 0 || stdout != "" || stderr != "" || report.Status != "success" {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q report=%+v", code, stdout, stderr, report)
|
||||
}
|
||||
raw, err := os.ReadFile(f.output)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Contains(raw, []byte("Hello world")) {
|
||||
t.Fatalf("correction missing: %s", raw)
|
||||
}
|
||||
calls := f.calls()
|
||||
if len(calls) < 3 || calls[0].Model != tc.proposal || calls[1].Model != tc.proposal {
|
||||
t.Fatalf("proposal models: %+v", calls)
|
||||
}
|
||||
for _, call := range calls[2:] {
|
||||
if !call.Validation || call.Model != tc.validation {
|
||||
t.Fatalf("validation model: %+v", calls)
|
||||
}
|
||||
}
|
||||
if report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 || report.LLMSummary.ProviderAttempts != uint64(len(calls)) {
|
||||
t.Fatalf("LLM summary: %+v", report.LLMSummary)
|
||||
}
|
||||
if report.Diagnostics == nil || report.Diagnostics.LLMExecutionPath == "" || report.Diagnostics.SelectedProfilesPath == "" {
|
||||
t.Fatalf("paths: %+v", report.Diagnostics)
|
||||
}
|
||||
})
|
||||
var document reporting.ErrorsDocument
|
||||
if err := yaml.Unmarshal(raw, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func TestSubprocessWarningEmptyRepeatedAndFailure(t *testing.T) {
|
||||
t.Run("domain rejection", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "low-confidence", "[grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].ValidatorRejected) == 0 {
|
||||
t.Fatalf("domain rejection: %d %q %+v", code, stderr, report)
|
||||
}
|
||||
if len(f.calls()) != 1 {
|
||||
t.Fatalf("semantic rejection retried: %+v", f.calls())
|
||||
}
|
||||
})
|
||||
t.Run("warning", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "warning", "[grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || report.Status != "success" || len(report.ModuleResults) != 1 || len(report.ModuleResults[0].Warnings) == 0 {
|
||||
t.Fatalf("warning result: %d %q %+v", code, stderr, report)
|
||||
}
|
||||
if len(f.calls()) != 2 || report.LLMSummary == nil || report.LLMSummary.StructuralRepairs != 1 {
|
||||
t.Fatalf("warning observations: %+v %+v", f.calls(), report.LLMSummary)
|
||||
}
|
||||
})
|
||||
t.Run("empty", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "noop", "[]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || len(f.calls()) != 0 || report.LLMSummary == nil || !report.LLMSummary.Usage.Total.Known || !report.LLMSummary.Usage.Total.Complete {
|
||||
t.Fatalf("empty result: %d %q %+v", code, stderr, report.LLMSummary)
|
||||
}
|
||||
})
|
||||
t.Run("repeated", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "noop", "[grammar, grammar]")
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 || stderr != "" || len(report.ModuleResults) != 2 || report.ModuleResults[0].ModuleInstance == report.ModuleResults[1].ModuleInstance || len(f.calls()) != 2 {
|
||||
t.Fatalf("repeated result: %d %q %+v %+v", code, stderr, report.ModuleResults, f.calls())
|
||||
}
|
||||
})
|
||||
t.Run("failure redaction", func(t *testing.T) {
|
||||
f := newProcessFixture(t, "failure", "[grammar]")
|
||||
code, stdout, stderr, report := f.run(t)
|
||||
if code != 1 || stdout != "" || report.Status != "failed" || report.ErrorPhase != "runner_execution" || report.LLMSummary == nil || report.LLMSummary.LogicalCalls != 1 {
|
||||
t.Fatalf("failure result: %d %q %q %+v", code, stdout, stderr, report)
|
||||
}
|
||||
if strings.Contains(stderr, f.provider.secret) {
|
||||
t.Fatal("secret in stderr")
|
||||
}
|
||||
for _, path := range []string{f.report, report.Diagnostics.DirectoryPath} {
|
||||
_ = filepath.Walk(path, func(name string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
content, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Contains(content, []byte(f.provider.secret)) {
|
||||
t.Fatalf("secret in %s", name)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if _, err := os.Stat(report.Diagnostics.LLMExecutionPath); err != nil {
|
||||
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
output, err := os.ReadFile(f.output)
|
||||
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||
t.Fatalf("output: %s %v", output, err)
|
||||
}
|
||||
document := f.errors(t)
|
||||
if document.Status != reporting.StatusCompleted || document.Summary.WorkUnitsCompleted == nil || *document.Summary.WorkUnitsCompleted != 1 || len(document.Errors) != 0 {
|
||||
t.Fatalf("errors document: %+v", document)
|
||||
}
|
||||
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
||||
if _, err := os.Stat(filepath.Join(f.dir, name)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
|
||||
t.Fatalf("debug created by default: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRedactsValidModelReasonsOnSuccessAndPartialFailure(t *testing.T) {
|
||||
for _, mode := range []string{"secret-rejection", "secret-partial-failure"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
f := newProcessFixture(t, mode, "[grammar, grammar]")
|
||||
f.provider.secret = `selected-credential-"\-123`
|
||||
f.provider.unselectedSecret = `unused-credential-"\-456`
|
||||
raw, err := os.ReadFile(f.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw = []byte(strings.Replace(string(raw), "backends:\n", "backends:\n unused:\n endpoint: "+f.server.URL+"/v1\n credential: AUDITA_TEST_UNUSED_KEY\n", 1))
|
||||
if err := os.WriteFile(f.config, raw, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, stderr, report := f.run(t)
|
||||
if mode == "secret-rejection" && (code != 0 || report.Status != "success") {
|
||||
t.Fatalf("code=%d %s", code, stderr)
|
||||
}
|
||||
if mode == "secret-partial-failure" {
|
||||
if code == 0 || report.Status != "failed" {
|
||||
t.Fatalf("expected failure: %d %+v", code, report)
|
||||
}
|
||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||
t.Fatalf("failure wrote complete output: %v", err)
|
||||
}
|
||||
}
|
||||
if len(report.ModuleResults) == 0 || len(report.ModuleResults[0].ValidatorRejected) == 0 {
|
||||
t.Fatalf("missing rejected domain result: %+v", report)
|
||||
}
|
||||
reason := report.ModuleResults[0].ValidatorRejected[0].Message
|
||||
if !strings.Contains(reason, "[REDACTED]") {
|
||||
t.Fatalf("reason not exercised: %s", reason)
|
||||
}
|
||||
sawLedger := false
|
||||
sawInteraction := false
|
||||
check := func(path string) {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
reports := filepath.Join(f.dir, "reports")
|
||||
if err := os.Mkdir(reports, 0700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
||||
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||
t.Fatalf("stdout run wrote transcript file: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(reports, "errors.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
||||
for _, debug := range []bool{false, true} {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
|
||||
if debug {
|
||||
extra = append(extra, "--debug")
|
||||
}
|
||||
code, stdout, stderr := f.run(t, true, extra...)
|
||||
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
||||
t.Fatalf("debug=%v code=%d stdout=%q stderr=%q", debug, code, stdout, stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||
t.Fatalf("disabled report created: %v", err)
|
||||
}
|
||||
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
||||
if err != nil || len(dirs) != boolInt(debug) {
|
||||
t.Fatalf("debug=%v dirs=%v %v", debug, dirs, err)
|
||||
}
|
||||
if debug {
|
||||
for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} {
|
||||
if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var walk func(any)
|
||||
walk = func(value any) {
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
for _, secret := range []string{f.provider.secret, f.provider.unselectedSecret} {
|
||||
escaped, _ := json.Marshal(secret)
|
||||
if strings.Contains(v, secret) || strings.Contains(v, string(escaped[1:len(escaped)-1])) {
|
||||
t.Fatalf("credential in %s", path)
|
||||
}
|
||||
}
|
||||
case []any:
|
||||
for _, item := range v {
|
||||
walk(item)
|
||||
}
|
||||
case map[string]any:
|
||||
for key, item := range v {
|
||||
walk(key)
|
||||
walk(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(path, ".json") {
|
||||
var value any
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
walk(value)
|
||||
} else {
|
||||
walk(string(raw))
|
||||
}
|
||||
if strings.HasSuffix(path, "correction-ledger.json") {
|
||||
sawLedger = true
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatal("ledger reason not exercised")
|
||||
}
|
||||
}
|
||||
if strings.HasSuffix(path, "response-payload.json") {
|
||||
sawInteraction = true
|
||||
}
|
||||
}
|
||||
check(f.report)
|
||||
err = filepath.WalkDir(filepath.Join(f.dir, "runs"), func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
check(path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sawLedger || !sawInteraction {
|
||||
t.Fatal("missing diagnostic artifacts")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessRepeatedModulesKeepSectionAttribution(t *testing.T) {
|
||||
f := newProcessFixture(t, "sections", "[grammar, grammar]")
|
||||
raw, err := os.ReadFile(f.config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
func boolInt(value bool) int {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
raw = append(raw, []byte("chunking:\n max_section_tokens: 3\n min_section_tokens: 1\n")...)
|
||||
if err := os.WriteFile(f.config, raw, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
return 0
|
||||
}
|
||||
|
||||
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
||||
f := newProcessFixture(t, "all-model-failure", "[grammar]")
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "1 work units with errors") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
input := `[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"},{"id":2,"speaker":"B","start":2,"end":3,"text":"good morning"}]`
|
||||
if err := os.WriteFile(f.transcript, []byte(input), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
output, err := os.ReadFile(f.output)
|
||||
if err != nil || !bytes.Contains(output, []byte("hello world")) {
|
||||
t.Fatalf("normalized output: %s %v", output, err)
|
||||
}
|
||||
code, _, stderr, report := f.run(t)
|
||||
if code != 0 {
|
||||
t.Fatalf("process: %d %s", code, stderr)
|
||||
document := f.errors(t)
|
||||
if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 1 {
|
||||
t.Fatalf("model failure report: %+v", document)
|
||||
}
|
||||
if len(report.ModuleResults) != 2 {
|
||||
t.Fatalf("module results: %+v", report)
|
||||
}
|
||||
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "*"))
|
||||
if err != nil || len(dirs) != 1 {
|
||||
t.Fatalf("run directories: %v %v", dirs, err)
|
||||
}
|
||||
raw, err = os.ReadFile(filepath.Join(dirs[0], "utilization-diagnostics.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var utilization reporting.UtilizationDiagnostics
|
||||
if err := json.Unmarshal(raw, &utilization); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(utilization.Modules) != 2 {
|
||||
t.Fatalf("spurious modules: %+v", utilization.Modules)
|
||||
}
|
||||
for i, module := range utilization.Modules {
|
||||
if module.ModuleInstance != report.ModuleResults[i].ModuleInstance || module.ProposalLLMCalls != 2 || module.ValidationLLMCalls != 2 {
|
||||
t.Fatalf("module attribution: %+v", utilization.Modules)
|
||||
}
|
||||
}
|
||||
raw, err = os.ReadFile(filepath.Join(dirs[0], "llm-execution.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var decoded struct {
|
||||
Calls []struct {
|
||||
Context struct {
|
||||
Role string `json:"role"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
SectionIndex *int `json:"section_index"`
|
||||
BatchIndex *int `json:"batch_index"`
|
||||
} `json:"context"`
|
||||
} `json:"calls"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &decoded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]map[int]bool{}
|
||||
for _, call := range decoded.Calls {
|
||||
c := call.Context
|
||||
if c.Role != "validation" {
|
||||
continue
|
||||
}
|
||||
if c.SectionIndex == nil || c.BatchIndex == nil || *c.BatchIndex != 0 {
|
||||
t.Fatalf("missing section/batch: %+v", c)
|
||||
}
|
||||
if seen[c.ModuleInstance] == nil {
|
||||
seen[c.ModuleInstance] = map[int]bool{}
|
||||
}
|
||||
seen[c.ModuleInstance][*c.SectionIndex] = true
|
||||
}
|
||||
for _, module := range utilization.Modules {
|
||||
if len(seen[module.ModuleInstance]) != 2 {
|
||||
t.Fatalf("section attribution: %+v", seen)
|
||||
}
|
||||
}
|
||||
responseCount := 0
|
||||
if err := filepath.WalkDir(dirs[0], func(path string, entry os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !entry.IsDir() && strings.Contains(path, "section-") && strings.Contains(path, "batch-") && strings.HasSuffix(path, "response-payload.json") {
|
||||
responseCount++
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if responseCount != 4 {
|
||||
t.Fatalf("validation artifact collision: %d", responseCount)
|
||||
}
|
||||
for _, module := range report.ModuleResults {
|
||||
if len(module.ValidatorRejected) != 2 || module.ValidatorRejected[0].ProposalIndex != 0 || module.ValidatorRejected[1].ProposalIndex != 1 {
|
||||
t.Fatalf("decision order: %+v", module.ValidatorRejected)
|
||||
}
|
||||
if strings.Contains(stderr, f.provider.secret) {
|
||||
t.Fatal("secret leaked in stderr")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
||||
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
||||
code, _, stderr := f.run(t, true)
|
||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
document := f.errors(t)
|
||||
if len(document.Errors) != 1 || document.Errors[0].ModuleInstance != "grammar_1" || *document.Summary.WorkUnitsCompleted != 2 {
|
||||
t.Fatalf("partial outcome: %+v", document)
|
||||
}
|
||||
output, err := os.ReadFile(f.output)
|
||||
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
||||
t.Fatalf("later correction: %s %v", output, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
code, _, _ := f.run(t, true)
|
||||
if code != 1 || f.provider.callCount() != 0 {
|
||||
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
||||
}
|
||||
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
||||
t.Fatalf("failed report: %+v", document)
|
||||
}
|
||||
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed transcript published: %v", err)
|
||||
}
|
||||
var recoveries reporting.RecoveriesDocument
|
||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if recoveries.Status != reporting.StatusFailed {
|
||||
t.Fatalf("recoveries status: %+v", recoveries)
|
||||
}
|
||||
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
||||
t.Fatalf("Markdown status: %s %v", raw, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||
code, _, stderr := f.run(t, true)
|
||||
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "report directory") {
|
||||
t.Fatalf("preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
||||
t.Fatalf("report initialized before preflight: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
||||
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
|
||||
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
if _, err := os.Stat(f.output); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if document := f.errors(t); document.Status != reporting.StatusCompleted {
|
||||
t.Fatalf("report status: %+v", document)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
f.output = filepath.Join(f.dir, "missing", "output.json")
|
||||
code, _, stderr := f.run(t, true, "--reporting=false")
|
||||
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
|
||||
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user