946 lines
36 KiB
Go
946 lines
36 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"os/exec"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
func TestProcessSubprocessHelper(t *testing.T) {
|
|
if os.Getenv("AUDITA_HELPER_PROCESS") != "1" {
|
|
return
|
|
}
|
|
for i, arg := range os.Args {
|
|
if arg == "--" {
|
|
if value := os.Getenv("AUDITA_TEST_UMASK"); value != "" {
|
|
mask, err := strconv.ParseUint(value, 8, 32)
|
|
if err != nil {
|
|
os.Exit(2)
|
|
}
|
|
syscall.Umask(int(mask))
|
|
}
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
code := cli.Run(ctx, os.Args[i+1:], os.Stdout, os.Stderr)
|
|
stop()
|
|
os.Exit(code)
|
|
}
|
|
}
|
|
os.Exit(2)
|
|
}
|
|
|
|
type fixtureProvider struct {
|
|
mu sync.Mutex
|
|
calls int
|
|
proposalCalls int
|
|
validationCalls int
|
|
mode string
|
|
secret string
|
|
entered chan struct{}
|
|
beforeResponse func()
|
|
}
|
|
|
|
func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
|
var request struct {
|
|
ResponseFormat struct {
|
|
JSONSchema struct {
|
|
Schema struct {
|
|
Properties map[string]any `json:"properties"`
|
|
} `json:"schema"`
|
|
} `json:"json_schema"`
|
|
} `json:"response_format"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
|
|
http.Error(w, "bad request", 400)
|
|
return
|
|
}
|
|
_, validation := request.ResponseFormat.JSONSchema.Schema.Properties["validations"]
|
|
p.mu.Lock()
|
|
p.calls++
|
|
if validation {
|
|
p.validationCalls++
|
|
} else {
|
|
p.proposalCalls++
|
|
}
|
|
validationCall := p.validationCalls
|
|
proposalCall := p.proposalCalls
|
|
beforeResponse := p.beforeResponse
|
|
p.mu.Unlock()
|
|
if beforeResponse != nil {
|
|
beforeResponse()
|
|
}
|
|
if p.mode == "wait-cancel" {
|
|
select {
|
|
case p.entered <- struct{}{}:
|
|
default:
|
|
}
|
|
<-r.Context().Done()
|
|
return
|
|
}
|
|
if p.mode == "all-model-failure" || (p.mode == "all-validation-failure" && validation) || (p.mode == "partial-validation" && validation && validationCall == 1) || (p.mode == "repair-filter-later-failure" && !validation && proposalCall == 3) {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
_, _ = fmt.Fprintf(w, `{"error":{"message":"provider echoed %s"}}`, p.secret)
|
|
return
|
|
}
|
|
content := `{"corrections":[{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
|
if p.mode == "repair-filter-later-failure" && !validation {
|
|
if proposalCall == 1 {
|
|
content = `{"wrong":true}`
|
|
} else {
|
|
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
|
}
|
|
}
|
|
if strings.HasPrefix(p.mode, "span-") {
|
|
switch p.mode {
|
|
case "span-clean", "span-filter":
|
|
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}`
|
|
if p.mode == "span-filter" {
|
|
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}`
|
|
}
|
|
case "span-spacing":
|
|
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}`
|
|
case "span-negative":
|
|
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}`
|
|
}
|
|
if validation {
|
|
if p.mode == "span-clean" || p.mode == "span-filter" {
|
|
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
|
} else {
|
|
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
|
}
|
|
if p.mode == "span-negative" && validationCall == 2 {
|
|
content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
|
|
}
|
|
}
|
|
}
|
|
if validation && !strings.HasPrefix(p.mode, "span-") {
|
|
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
|
if p.mode == "repair-filter-later-failure" {
|
|
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
|
} else if p.mode == "invalid-decision-after-repair" {
|
|
if validationCall == 1 {
|
|
content = `{"wrong":true}`
|
|
} else {
|
|
content = `{"validations":[{"correction_index":9,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
|
}
|
|
}
|
|
}
|
|
encoded, _ := json.Marshal(content)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
_, _ = fmt.Fprintf(w, `{"model":"synthetic","choices":[{"message":{"content":%s}}],"usage":{"prompt_tokens":2,"completion_tokens":3,"total_tokens":5}}`, encoded)
|
|
}
|
|
|
|
func (p *fixtureProvider) callCount() int { p.mu.Lock(); defer p.mu.Unlock(); return p.calls }
|
|
|
|
type processFixture struct {
|
|
dir string
|
|
config string
|
|
transcript string
|
|
glossary string
|
|
output string
|
|
provider *fixtureProvider
|
|
server *httptest.Server
|
|
}
|
|
|
|
func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
p := &fixtureProvider{mode: mode, secret: `selected-secret-"\-123`, entered: make(chan struct{}, 1)}
|
|
server := httptest.NewServer(http.HandlerFunc(p.serve))
|
|
t.Cleanup(server.Close)
|
|
if err := os.Mkdir(filepath.Join(dir, "profiles"), 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, id := range []string{"proposal", "validation"} {
|
|
profile := fmt.Sprintf("id: %s\nbackend: local\nmodel: local\ntimeout_seconds: 5\n", id)
|
|
if err := os.WriteFile(filepath.Join(dir, "profiles", id+".yaml"), []byte(profile), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
configPath := filepath.Join(dir, "config.yml")
|
|
repairs := 0
|
|
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
|
repairs = 1
|
|
}
|
|
if strings.HasPrefix(mode, "span-") {
|
|
repairs = 1
|
|
}
|
|
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
|
if strings.HasPrefix(mode, "span-") {
|
|
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
|
}
|
|
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
transcript := filepath.Join(dir, "input.json")
|
|
if err := os.WriteFile(transcript, []byte(`[{"id":1,"speaker":"A","start":0,"end":1,"text":"hello world"}]`), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
glossary := filepath.Join(dir, "glossary.yaml")
|
|
if err := os.WriteFile(glossary, []byte("glossary:\n - name: Example\n category: product\n summary: A sample.\n"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return processFixture{dir: dir, config: configPath, transcript: transcript, glossary: glossary,
|
|
output: filepath.Join(dir, "output.json"), provider: p, server: server}
|
|
}
|
|
|
|
func (f processFixture) run(t *testing.T, includeOutput bool, extra ...string) (int, string, string) {
|
|
t.Helper()
|
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary}
|
|
if includeOutput {
|
|
args = append(args, "--output", f.output)
|
|
}
|
|
args = append(args, extra...)
|
|
command := exec.Command(os.Args[0], args...)
|
|
command.Dir = f.dir
|
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
|
var stdout, stderr bytes.Buffer
|
|
command.Stdout, command.Stderr = &stdout, &stderr
|
|
err := command.Run()
|
|
code := 0
|
|
if err != nil {
|
|
if exit, ok := err.(*exec.ExitError); ok {
|
|
code = exit.ExitCode()
|
|
} else {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return code, stdout.String(), stderr.String()
|
|
}
|
|
|
|
func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(filepath.Join(f.dir, "errors.yml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var document reporting.ErrorsDocument
|
|
if err := yaml.Unmarshal(raw, &document); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return document
|
|
}
|
|
|
|
func (f processFixture) recoveries(t *testing.T) reporting.RecoveriesDocument {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var document reporting.RecoveriesDocument
|
|
if err := yaml.Unmarshal(raw, &document); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return document
|
|
}
|
|
|
|
func (f processFixture) setSegments(t *testing.T, texts ...string) {
|
|
t.Helper()
|
|
segments := make([]schema.Segment, len(texts))
|
|
for i, value := range texts {
|
|
segments[i] = schema.Segment{ID: i + 1, Speaker: "A", Start: float64(i * 10), End: float64(i*10 + 1), Text: value, Categories: []string{"dialogue"}}
|
|
}
|
|
raw, err := json.Marshal(segments)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(f.transcript, raw, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transcript {
|
|
t.Helper()
|
|
raw, err := os.ReadFile(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
transcript, err := schema.ParseTranscriptJSON(raw)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return transcript
|
|
}
|
|
|
|
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {
|
|
t.Helper()
|
|
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
|
if err != nil || len(dirs) != 1 {
|
|
t.Fatalf("expected one debug run: %v %v", dirs, err)
|
|
}
|
|
normalized := f.readTranscript(t, filepath.Join(dirs[0], "normalized-transcript.json"))
|
|
raw, err := os.ReadFile(filepath.Join(dirs[0], "correction-ledger.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var ledger []map[string]json.RawMessage
|
|
if err := json.Unmarshal(raw, &ledger); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return normalized, ledger
|
|
}
|
|
|
|
func assertSpanMetadata(t *testing.T, normalized, output *schema.Transcript) {
|
|
t.Helper()
|
|
if len(normalized.Segments) != len(output.Segments) {
|
|
t.Fatalf("segment membership changed: normalized=%+v output=%+v", normalized, output)
|
|
}
|
|
for i := range normalized.Segments {
|
|
normalized.Segments[i].Text = output.Segments[i].Text
|
|
}
|
|
if !reflect.DeepEqual(normalized, output) {
|
|
t.Fatalf("segment metadata/order changed: normalized=%+v output=%+v", normalized, output)
|
|
}
|
|
}
|
|
|
|
func ledgerDisposition(entry map[string]json.RawMessage) string {
|
|
var value string
|
|
_ = json.Unmarshal(entry["disposition"], &value)
|
|
return value
|
|
}
|
|
|
|
func ledgerHasDecision(t *testing.T, entry map[string]json.RawMessage, field, key string, approved bool) bool {
|
|
t.Helper()
|
|
var decisions []struct {
|
|
ValidatorKey string `json:"validator_key"`
|
|
Approved bool `json:"approved"`
|
|
}
|
|
if err := json.Unmarshal(entry[field], &decisions); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, decision := range decisions {
|
|
if decision.ValidatorKey == key && decision.Approved == approved {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
|
|
t.Helper()
|
|
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
|
if err != nil || len(dirs) != 1 {
|
|
t.Fatalf("debug run: %v %v", dirs, err)
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(dirs[0], "llm-execution.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var execution struct {
|
|
Calls []struct {
|
|
Summary struct {
|
|
ProviderAttempts int `json:"provider_attempts"`
|
|
StructuralRepairs int `json:"structural_repairs"`
|
|
} `json:"summary"`
|
|
} `json:"calls"`
|
|
}
|
|
if err := json.Unmarshal(raw, &execution); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(execution.Calls) != wantCalls || f.provider.callCount() != wantCalls {
|
|
t.Fatalf("unexpected generation counts: calls=%d provider=%d", len(execution.Calls), f.provider.callCount())
|
|
}
|
|
for _, call := range execution.Calls {
|
|
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
|
|
t.Fatalf("unexpected structural repair: %+v", call)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
|
|
f := newProcessFixture(t, "span-clean", "[spoken_word]")
|
|
f.setSegments(t, "uh hello", "I I think")
|
|
code, stdout, stderr := f.run(t, true, "--debug")
|
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 0 recoveries") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
output := f.readTranscript(t, f.output)
|
|
normalized, ledger := f.debugArtifacts(t)
|
|
assertSpanMetadata(t, normalized, output)
|
|
if len(output.Segments) != 2 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" {
|
|
t.Fatalf("unexpected corrected transcript: %+v", output)
|
|
}
|
|
if len(ledger) != 2 || ledgerDisposition(ledger[0]) != "applied" || ledgerDisposition(ledger[1]) != "applied" || string(ledger[0]["proposed_corrected_text"]) != `""` || string(ledger[0]["applied_corrected_text"]) != `""` {
|
|
t.Fatalf("deletion missing from ledger: %+v", ledger)
|
|
}
|
|
errors, recoveries := f.errors(t), f.recoveries(t)
|
|
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 0 || *errors.Summary.CorrectionsApplied != 2 || *errors.Summary.WorkUnitsCompleted != 1 || *recoveries.Summary.CorrectionsApplied != 2 {
|
|
t.Fatalf("report totals disagree: errors=%+v recoveries=%+v", errors, recoveries)
|
|
}
|
|
f.assertNoStructuralRepairs(t, 3)
|
|
}
|
|
|
|
func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) {
|
|
f := newProcessFixture(t, "span-spacing", "[grammar]")
|
|
f.setSegments(t, "hello world")
|
|
code, _, stderr := f.run(t, true, "--debug")
|
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
output := f.readTranscript(t, f.output)
|
|
normalized, _ := f.debugArtifacts(t)
|
|
assertSpanMetadata(t, normalized, output)
|
|
if len(output.Segments) != 1 || output.Segments[0].Text != "hello world" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
|
|
t.Fatalf("spacing correction failed: %+v", output)
|
|
}
|
|
f.assertNoStructuralRepairs(t, 3)
|
|
}
|
|
|
|
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) {
|
|
f := newProcessFixture(t, "span-filter", "[spoken_word]")
|
|
f.setSegments(t, "uh hello", "I I think", "uh")
|
|
code, _, stderr := f.run(t, true, "--debug")
|
|
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 1 recoveries") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
output := f.readTranscript(t, f.output)
|
|
normalized, ledger := f.debugArtifacts(t)
|
|
assertSpanMetadata(t, normalized, output)
|
|
if len(output.Segments) != 3 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" || output.Segments[2].Text != "uh" {
|
|
t.Fatalf("filtered segment or survivors wrong: %+v", output)
|
|
}
|
|
recoveries := f.recoveries(t)
|
|
if len(f.errors(t).Errors) != 0 || len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].Reasons) != 1 || recoveries.Recoveries[0].Reasons[0] != "proposals_filtered" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || recoveries.Recoveries[0].FilteredProposals[0].Reason != "empty_resulting_segment" || *recoveries.Summary.ProposalResponsesWithFiltering != 1 {
|
|
t.Fatalf("filtering recovery wrong: %+v", recoveries)
|
|
}
|
|
if len(ledger) != 3 || ledgerDisposition(ledger[2]) != "rejected" || string(ledger[2]["proposed_corrected_text"]) != `""` {
|
|
t.Fatalf("filtered deletion evidence missing: %+v", ledger)
|
|
}
|
|
if !ledgerHasDecision(t, ledger[2], "deterministic_validator_decisions", "non_empty_resulting_segment", false) {
|
|
t.Fatalf("renamed validator missing from evidence: %+v", ledger[2])
|
|
}
|
|
f.assertNoStructuralRepairs(t, 3)
|
|
}
|
|
|
|
func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
|
|
f := newProcessFixture(t, "span-negative", "[grammar]")
|
|
f.setSegments(t, "not good")
|
|
code, _, stderr := f.run(t, true, "--debug")
|
|
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 0 work units with errors; 0 recoveries") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
output := f.readTranscript(t, f.output)
|
|
normalized, ledger := f.debugArtifacts(t)
|
|
assertSpanMetadata(t, normalized, output)
|
|
if len(output.Segments) != 1 || output.Segments[0].Text != "not good" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
|
|
t.Fatalf("negative review changed outcome: %+v", output)
|
|
}
|
|
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "rejected" || string(ledger[0]["proposed_corrected_text"]) != `""` {
|
|
t.Fatalf("rejection evidence missing: %+v", ledger)
|
|
}
|
|
if _, present := ledger[0]["applied_corrected_text"]; present || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "editorial_review", true) || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "meaning_reversal_review", false) {
|
|
t.Fatalf("negative decision not recorded: %+v", ledger[0])
|
|
}
|
|
f.assertNoStructuralRepairs(t, 3)
|
|
}
|
|
|
|
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
code, stdout, stderr := f.run(t, true)
|
|
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed: 1 corrections applied; 0 work units with errors; 0 recoveries.") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
|
t.Fatalf("output: %s %v", output, err)
|
|
}
|
|
document := f.errors(t)
|
|
if document.Status != reporting.StatusCompleted || document.Summary.WorkUnitsCompleted == nil || *document.Summary.WorkUnitsCompleted != 1 || len(document.Errors) != 0 {
|
|
t.Fatalf("errors document: %+v", document)
|
|
}
|
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
|
report, err := os.ReadFile(filepath.Join(f.dir, name))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if bytes.Contains(report, []byte("hello world")) || bytes.Contains(report, []byte("Hello world")) || bytes.Contains(report, []byte(f.provider.secret)) {
|
|
t.Fatalf("ordinary report contains payload or secret: %s", name)
|
|
}
|
|
}
|
|
if _, err := os.Stat(filepath.Join(f.dir, "runs")); !os.IsNotExist(err) {
|
|
t.Fatalf("debug created by default: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessStdoutAndSelectedReportDirectory(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
reports := filepath.Join(f.dir, "reports")
|
|
if err := os.Mkdir(reports, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, stdout, stderr := f.run(t, false, "--report-dir", reports)
|
|
if code != 0 || !strings.Contains(stdout, "Hello world") || !strings.HasPrefix(stderr, "Completed:") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
|
t.Fatalf("stdout run wrote transcript file: %v", err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(reports, "errors.yml")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessReportingAndDebugSelections(t *testing.T) {
|
|
var expectedOutput, expectedSummary string
|
|
for _, selection := range []struct{ reporting, debug bool }{
|
|
{false, false}, {false, true}, {true, false}, {true, true},
|
|
} {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
extra := []string{"--reporting=false", "--report-dir", filepath.Join(f.dir, "missing")}
|
|
if selection.reporting {
|
|
extra = []string{"--reporting=true", "--report-dir", filepath.Join(f.dir, "missing")}
|
|
}
|
|
if selection.debug {
|
|
extra = append(extra, "--debug")
|
|
}
|
|
code, stdout, stderr := f.run(t, true, extra...)
|
|
if code != 0 || stdout != "" || !strings.HasPrefix(stderr, "Completed:") {
|
|
t.Fatalf("selection=%+v code=%d stdout=%q stderr=%q", selection, code, stdout, stderr)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
summary := strings.SplitN(stderr, "\n", 2)[0]
|
|
if expectedOutput == "" {
|
|
expectedOutput, expectedSummary = string(output), summary
|
|
} else if string(output) != expectedOutput || summary != expectedSummary {
|
|
t.Fatalf("selection changed processing: %+v summary=%q output=%s", selection, summary, output)
|
|
}
|
|
for _, name := range []string{"errors.yml", "recoveries.yml", "report.md"} {
|
|
_, err := os.Stat(filepath.Join(f.dir, name))
|
|
if selection.reporting && err != nil || !selection.reporting && !os.IsNotExist(err) {
|
|
t.Fatalf("selection=%+v report %s: %v", selection, name, err)
|
|
}
|
|
}
|
|
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
|
if err != nil || len(dirs) != boolInt(selection.debug) {
|
|
t.Fatalf("selection=%+v dirs=%v %v", selection, dirs, err)
|
|
}
|
|
if selection.debug {
|
|
for _, name := range []string{"source-transcript.json", "normalized-transcript.json", "correction-ledger.json", "llm-execution.json"} {
|
|
if _, err := os.Stat(filepath.Join(dirs[0], name)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func boolInt(value bool) int {
|
|
if value {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func TestSubprocessAllModelFailureStillCompletes(t *testing.T) {
|
|
f := newProcessFixture(t, "all-model-failure", "[grammar, grammar]")
|
|
code, stdout, stderr := f.run(t, true)
|
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("hello world")) {
|
|
t.Fatalf("normalized output: %s %v", output, err)
|
|
}
|
|
document := f.errors(t)
|
|
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || document.Errors[0].Scope != "work_unit" || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 2 {
|
|
t.Fatalf("model failure report: %+v", document)
|
|
}
|
|
if strings.Contains(stderr, f.provider.secret) {
|
|
t.Fatal("secret leaked in stderr")
|
|
}
|
|
}
|
|
|
|
func TestSubprocessAllValidationFailureStillCompletes(t *testing.T) {
|
|
f := newProcessFixture(t, "all-validation-failure", "[grammar, grammar]")
|
|
code, stdout, stderr := f.run(t, true)
|
|
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 work units with errors") {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
|
|
t.Fatalf("unchecked output: %s %v", output, err)
|
|
}
|
|
document := f.errors(t)
|
|
if document.Status != reporting.StatusCompleted || len(document.Errors) != 2 || *document.Summary.WorkUnitsWithErrors != 2 || document.Errors[0].ModuleInstance == document.Errors[1].ModuleInstance || f.provider.callCount() != 4 {
|
|
t.Fatalf("validation failure report: %+v; calls=%d", document, f.provider.callCount())
|
|
}
|
|
}
|
|
|
|
func TestSubprocessRecoveredFilteringSurvivesLaterFailure(t *testing.T) {
|
|
f := newProcessFixture(t, "repair-filter-later-failure", "[grammar, grammar]")
|
|
code, _, stderr := f.run(t, true)
|
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors; 1 recoveries") {
|
|
t.Fatalf("code=%d stderr=%q errors=%+v", code, stderr, f.errors(t))
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
|
t.Fatalf("output: %s %v", output, err)
|
|
}
|
|
errors := f.errors(t)
|
|
if len(errors.Errors) != 1 || errors.Errors[0].ModuleInstance != "grammar_2" {
|
|
t.Fatalf("later failure: %+v", errors)
|
|
}
|
|
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var recoveries reporting.RecoveriesDocument
|
|
if err := yaml.Unmarshal(raw, &recoveries); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].ModuleInstance != "grammar_1" || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || *recoveries.Summary.Recoveries != 1 {
|
|
t.Fatalf("combined recovery: %+v", recoveries)
|
|
}
|
|
if !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "output_repaired") || !strings.Contains(strings.Join(recoveries.Recoveries[0].Reasons, ","), "proposals_filtered") {
|
|
t.Fatalf("recovery reasons: %+v", recoveries.Recoveries[0])
|
|
}
|
|
}
|
|
|
|
func TestSubprocessRepairedInvalidDecisionsRemainAnError(t *testing.T) {
|
|
f := newProcessFixture(t, "invalid-decision-after-repair", "[grammar]")
|
|
code, _, stderr := f.run(t, true)
|
|
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 1 work units with errors; 0 recoveries") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
document := f.errors(t)
|
|
if document.Status != reporting.StatusCompleted || len(document.Errors) != 1 || len(document.Errors[0].Failures) != 1 || document.Errors[0].Failures[0].Reason != "validation_decisions_invalid" {
|
|
t.Fatalf("invalid decisions report: %+v", document)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("hello world")) || bytes.Contains(output, []byte("Hello world")) {
|
|
t.Fatalf("unchecked output: %s %v", output, err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessCancellationLeavesFailedReportsWithoutTranscript(t *testing.T) {
|
|
f := newProcessFixture(t, "wait-cancel", "[grammar, grammar]")
|
|
ctx, stop := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer stop()
|
|
args := []string{"-test.run=^TestProcessSubprocessHelper$", "--", "process", f.transcript, "--config", f.config, "--glossary", f.glossary, "--output", f.output}
|
|
command := exec.CommandContext(ctx, os.Args[0], args...)
|
|
command.Env = append(os.Environ(), "AUDITA_HELPER_PROCESS=1", "AUDITA_TEST_PROVIDER_KEY="+f.provider.secret)
|
|
var stderr bytes.Buffer
|
|
command.Stderr = &stderr
|
|
if err := command.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-f.provider.entered:
|
|
case <-ctx.Done():
|
|
t.Fatalf("provider was not reached: %v", ctx.Err())
|
|
}
|
|
if err := command.Process.Signal(os.Interrupt); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := command.Wait()
|
|
if ctx.Err() != nil || err == nil {
|
|
t.Fatalf("cancellation did not stop run: %v %v stderr=%q", err, ctx.Err(), stderr.String())
|
|
}
|
|
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) == 0 {
|
|
t.Fatalf("cancellation report: %+v", document)
|
|
}
|
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
|
t.Fatalf("canceled run published transcript: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessPartialValidationAndLaterStage(t *testing.T) {
|
|
f := newProcessFixture(t, "partial-validation", "[grammar, grammar]")
|
|
code, _, stderr := f.run(t, true)
|
|
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 1 work units with errors") {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
document := f.errors(t)
|
|
if len(document.Errors) != 1 || document.Errors[0].ModuleInstance != "grammar_1" || *document.Summary.WorkUnitsCompleted != 2 {
|
|
t.Fatalf("partial outcome: %+v", document)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Contains(output, []byte("Hello world")) {
|
|
t.Fatalf("later correction: %s %v", output, err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessInvalidInputLeavesFailedReports(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
if err := os.WriteFile(f.transcript, []byte("not JSON"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, _, _ := f.run(t, true)
|
|
if code != 1 || f.provider.callCount() != 0 {
|
|
t.Fatalf("invalid input dispatched: code=%d calls=%d", code, f.provider.callCount())
|
|
}
|
|
if document := f.errors(t); document.Status != reporting.StatusFailed || len(document.Errors) != 1 || document.Errors[0].Scope != "run" {
|
|
t.Fatalf("failed report: %+v", document)
|
|
}
|
|
if _, err := os.Stat(f.output); !os.IsNotExist(err) {
|
|
t.Fatalf("failed transcript published: %v", err)
|
|
}
|
|
var recoveries reporting.RecoveriesDocument
|
|
if raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml")); err != nil {
|
|
t.Fatal(err)
|
|
} else if err := yaml.Unmarshal(raw, &recoveries); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if recoveries.Status != reporting.StatusFailed {
|
|
t.Fatalf("recoveries status: %+v", recoveries)
|
|
}
|
|
if raw, err := os.ReadFile(filepath.Join(f.dir, "report.md")); err != nil || !bytes.Contains(raw, []byte("Status: failed")) {
|
|
t.Fatalf("Markdown status: %s %v", raw, err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessOutputFailureBeforeModelDispatch(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
f.output = filepath.Join(f.dir, "missing", "output.json")
|
|
code, _, stderr := f.run(t, true)
|
|
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "report directory") {
|
|
t.Fatalf("preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(f.dir, "errors.yml")); !os.IsNotExist(err) {
|
|
t.Fatalf("report initialized before preflight: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessOptionalDebugFailureDoesNotChangeOutcome(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", f.transcript)
|
|
if code != 0 || !strings.HasPrefix(stderr, "Completed: 1 corrections applied") || !strings.Contains(stderr, "debug directory") {
|
|
t.Fatalf("optional debug failure changed processing: code=%d stderr=%q", code, stderr)
|
|
}
|
|
if _, err := os.Stat(f.output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if document := f.errors(t); document.Status != reporting.StatusCompleted {
|
|
t.Fatalf("report status: %+v", document)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessTranscriptPreflightWithoutReporting(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
f.output = filepath.Join(f.dir, "missing", "output.json")
|
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
|
if code == 0 || f.provider.callCount() != 0 || !strings.Contains(stderr, "transcript output parent") {
|
|
t.Fatalf("transcript preflight: code=%d calls=%d stderr=%q", code, f.provider.callCount(), stderr)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessOutputPermissionsHonorUmask(t *testing.T) {
|
|
for _, mask := range []string{"0077", "0022"} {
|
|
t.Run(mask, func(t *testing.T) {
|
|
t.Setenv("AUDITA_TEST_UMASK", mask)
|
|
f := newProcessFixture(t, "normal", "[]")
|
|
paths := []string{f.output, filepath.Join(f.dir, "errors.yml"), filepath.Join(f.dir, "recoveries.yml"), filepath.Join(f.dir, "report.md")}
|
|
parsed, _ := strconv.ParseUint(mask, 8, 32)
|
|
want := os.FileMode(0644 &^ parsed)
|
|
for attempt := 0; attempt < 2; attempt++ {
|
|
code, _, stderr := f.run(t, true)
|
|
if code != 0 {
|
|
t.Fatalf("run: %d %s", code, stderr)
|
|
}
|
|
for _, path := range paths {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if info.Mode().Perm() != want {
|
|
t.Errorf("%s mode=%o want=%o", filepath.Base(path), info.Mode().Perm(), want)
|
|
}
|
|
if err := os.Chmod(path, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
want = 0600 // Existing restrictive modes survive replacement under either mask.
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSubprocessWhitespaceOutputIsALiteralFile(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[]")
|
|
f.output = " "
|
|
code, stdout, stderr := f.run(t, true)
|
|
if code != 0 || stdout != "" {
|
|
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
|
}
|
|
output, err := os.ReadFile(filepath.Join(f.dir, " "))
|
|
if err != nil || !bytes.Contains(output, []byte("hello world")) {
|
|
t.Fatalf("missing transcript: %s %v", output, err)
|
|
}
|
|
if f.errors(t).Status != reporting.StatusCompleted {
|
|
t.Fatal("reports did not follow file output")
|
|
}
|
|
}
|
|
|
|
func TestSubprocessPathAnnouncementsAreRedacted(t *testing.T) {
|
|
for _, fail := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(fail), func(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[]")
|
|
selected, unused := "selected-path-secret", "unused-path-secret"
|
|
f.provider.secret = selected
|
|
t.Setenv("AUDITA_UNUSED_PATH_KEY", unused)
|
|
cfg, err := os.ReadFile(f.config)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg = bytes.Replace(cfg, []byte("backends:\n"), []byte("backends:\n unused:\n endpoint: http://unused.invalid/v1\n credential: AUDITA_UNUSED_PATH_KEY\n"), 1)
|
|
if err := os.WriteFile(f.config, cfg, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reportDir := filepath.Join(f.dir, unused)
|
|
if err := os.Mkdir(reportDir, 0700); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.output = filepath.Join(reportDir, "out.json")
|
|
debugParent := filepath.Join(f.dir, selected)
|
|
if fail {
|
|
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
code, _, stderr := f.run(t, true, "--debug", "--work-dir", debugParent)
|
|
if (!fail && code != 0) || (fail && code != 1) {
|
|
t.Fatalf("code=%d stderr=%q", code, stderr)
|
|
}
|
|
if strings.Contains(stderr, selected) || strings.Contains(stderr, unused) {
|
|
t.Fatalf("credential in stderr: %q", stderr)
|
|
}
|
|
if !strings.Contains(stderr, "[REDACTED]") {
|
|
t.Fatalf("missing redacted announcement: %q", stderr)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(reportDir, "report.md")); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
dirs, err := os.ReadDir(debugParent)
|
|
if err != nil || len(dirs) != 1 {
|
|
t.Fatalf("debug location: %v %v", dirs, err)
|
|
}
|
|
if !fail {
|
|
if _, err := os.Stat(f.output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSubprocessInteractionWriteFailureIsVisibleAndNonfatal(t *testing.T) {
|
|
f := newProcessFixture(t, "normal", "[grammar]")
|
|
code, _, baselineSummary := f.run(t, true)
|
|
if code != 0 {
|
|
t.Fatal(baselineSummary)
|
|
}
|
|
baseline, err := os.ReadFile(f.output)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.provider.mu.Lock()
|
|
f.provider.beforeResponse = func() {
|
|
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
|
if err != nil || len(dirs) != 1 {
|
|
t.Errorf("debug directory: %v %v", dirs, err)
|
|
return
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dirs[0], "grammar"), []byte("block interaction writes"), 0600); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
f.provider.mu.Unlock()
|
|
code, _, stderr := f.run(t, true, "--debug")
|
|
if code != 0 || !strings.HasPrefix(stderr, strings.SplitN(baselineSummary, "\n", 2)[0]) {
|
|
t.Fatalf("changed outcome: %d %q", code, stderr)
|
|
}
|
|
if !strings.Contains(stderr, "debug proposal interaction") || !strings.Contains(stderr, "debug validation interaction") {
|
|
t.Fatalf("missing debug failures: %q", stderr)
|
|
}
|
|
output, err := os.ReadFile(f.output)
|
|
if err != nil || !bytes.Equal(output, baseline) {
|
|
t.Fatalf("changed transcript: %s %v", output, err)
|
|
}
|
|
document := f.errors(t)
|
|
if len(document.Errors) != 0 || *document.Summary.Recoveries != 0 {
|
|
t.Fatalf("debug failure classified as model incident: %+v", document)
|
|
}
|
|
}
|
|
|
|
func TestSubprocessInPlaceTranscript(t *testing.T) {
|
|
for _, invalid := range []bool{false, true} {
|
|
t.Run(fmt.Sprint(invalid), func(t *testing.T) {
|
|
f := newProcessFixture(t, "success", "[grammar]")
|
|
f.output = f.transcript
|
|
if invalid {
|
|
if err := os.WriteFile(f.transcript, []byte("invalid"), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
before, err := os.ReadFile(f.transcript)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, stdout, stderr := f.run(t, true)
|
|
after, err := os.ReadFile(f.transcript)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if invalid {
|
|
if code != 1 || !bytes.Equal(before, after) {
|
|
t.Fatalf("failed processing changed input: code=%d %s", code, stderr)
|
|
}
|
|
} else if code != 0 || stdout != "" || !bytes.Contains(after, []byte("Hello world")) {
|
|
t.Fatalf("in-place publication failed: code=%d %s %s", code, stderr, after)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSubprocessProfileSources(t *testing.T) {
|
|
f := newProcessFixture(t, "success", "[]")
|
|
// PromptKit uses profile IDs, not filename stems. A report-named profile is
|
|
// still an input and must not be overwritten during report initialization.
|
|
loaded := filepath.Join(f.dir, "profiles", "errors.yml")
|
|
if err := os.Rename(filepath.Join(f.dir, "profiles", "proposal.yaml"), loaded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, err := os.ReadFile(loaded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.output = filepath.Join(f.dir, "profiles", "proposal.yml")
|
|
code, _, stderr := f.run(t, true, "--reporting=false")
|
|
if code != 0 {
|
|
t.Fatalf("unused alternate rejected: %d %s", code, stderr)
|
|
}
|
|
// Remove the transcript from the profile directory before the next load.
|
|
if err := os.Remove(f.output); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
code, _, stderr = f.run(t, true)
|
|
after, err := os.ReadFile(loaded)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if code != 2 || !bytes.Equal(before, after) || f.provider.callCount() != 0 {
|
|
t.Fatalf("loaded profile not protected: %d %s", code, stderr)
|
|
}
|
|
}
|