Verify span deletion through complete CLI workflows
This commit is contained in:
@@ -11,6 +11,7 @@ import (
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/cli"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -106,7 +108,30 @@ func (p *fixtureProvider) serve(w http.ResponseWriter, r *http.Request) {
|
||||
content = `{"corrections":[{"id":1,"original_text":"absent text","corrected_text":"X","confidence":0.99},{"id":1,"original_text":"hello world","corrected_text":"Hello world","confidence":0.99}]}`
|
||||
}
|
||||
}
|
||||
if validation {
|
||||
if strings.HasPrefix(p.mode, "span-") {
|
||||
switch p.mode {
|
||||
case "span-clean", "span-filter":
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99}]}`
|
||||
if p.mode == "span-filter" {
|
||||
content = `{"corrections":[{"id":1,"original_text":"uh ","corrected_text":"","confidence":0.99},{"id":2,"original_text":"I I","corrected_text":"I","confidence":0.99},{"id":3,"original_text":"uh","corrected_text":"","confidence":0.99}]}`
|
||||
}
|
||||
case "span-spacing":
|
||||
content = `{"corrections":[{"id":1,"original_text":" ","corrected_text":" ","confidence":0.99}]}`
|
||||
case "span-negative":
|
||||
content = `{"corrections":[{"id":1,"original_text":"not ","corrected_text":"","confidence":0.99}]}`
|
||||
}
|
||||
if validation {
|
||||
if p.mode == "span-clean" || p.mode == "span-filter" {
|
||||
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"},{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
} else {
|
||||
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
}
|
||||
if p.mode == "span-negative" && validationCall == 2 {
|
||||
content = `{"validations":[{"correction_index":0,"approved":false,"confidence":0.99,"reason":"would reverse meaning"}]}`
|
||||
}
|
||||
}
|
||||
}
|
||||
if validation && !strings.HasPrefix(p.mode, "span-") {
|
||||
content = `{"validations":[{"correction_index":0,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
if p.mode == "repair-filter-later-failure" {
|
||||
content = `{"validations":[{"correction_index":1,"approved":true,"confidence":0.99,"reason":"ok"}]}`
|
||||
@@ -155,7 +180,13 @@ func newProcessFixture(t *testing.T, mode, modules string) processFixture {
|
||||
if mode == "repair-filter-later-failure" || mode == "invalid-decision-after-repair" {
|
||||
repairs = 1
|
||||
}
|
||||
if strings.HasPrefix(mode, "span-") {
|
||||
repairs = 1
|
||||
}
|
||||
configText := fmt.Sprintf("version: 2\nbackends:\n local:\n endpoint: %s/v1\n credential: AUDITA_TEST_PROVIDER_KEY\nllm:\n proposal_profile: proposal\n validation_profile: validation\n transient_retries: 0\n repair_attempts: %d\npipeline:\n modules: %s\ndiagnostics:\n work_dir: %s\n", server.URL, repairs, modules, filepath.Join(dir, "runs"))
|
||||
if strings.HasPrefix(mode, "span-") {
|
||||
configText += "validation:\n max_prompt_tokens: 4096\nchunking:\n target_sections: 1\n max_section_tokens: 8192\n min_section_tokens: 1\nnormalization:\n max_segment_gap: 0\n"
|
||||
}
|
||||
if err := os.WriteFile(configPath, []byte(configText), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -208,6 +239,219 @@ func (f processFixture) errors(t *testing.T) reporting.ErrorsDocument {
|
||||
return document
|
||||
}
|
||||
|
||||
func (f processFixture) recoveries(t *testing.T) reporting.RecoveriesDocument {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(filepath.Join(f.dir, "recoveries.yml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var document reporting.RecoveriesDocument
|
||||
if err := yaml.Unmarshal(raw, &document); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return document
|
||||
}
|
||||
|
||||
func (f processFixture) setSegments(t *testing.T, texts ...string) {
|
||||
t.Helper()
|
||||
segments := make([]schema.Segment, len(texts))
|
||||
for i, value := range texts {
|
||||
segments[i] = schema.Segment{ID: i + 1, Speaker: "A", Start: float64(i * 10), End: float64(i*10 + 1), Text: value, Categories: []string{"dialogue"}}
|
||||
}
|
||||
raw, err := json.Marshal(segments)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(f.transcript, raw, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (f processFixture) readTranscript(t *testing.T, path string) *schema.Transcript {
|
||||
t.Helper()
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
transcript, err := schema.ParseTranscriptJSON(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return transcript
|
||||
}
|
||||
|
||||
func (f processFixture) debugArtifacts(t *testing.T) (*schema.Transcript, []map[string]json.RawMessage) {
|
||||
t.Helper()
|
||||
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
||||
if err != nil || len(dirs) != 1 {
|
||||
t.Fatalf("expected one debug run: %v %v", dirs, err)
|
||||
}
|
||||
normalized := f.readTranscript(t, filepath.Join(dirs[0], "normalized-transcript.json"))
|
||||
raw, err := os.ReadFile(filepath.Join(dirs[0], "correction-ledger.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ledger []map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &ledger); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return normalized, ledger
|
||||
}
|
||||
|
||||
func assertSpanMetadata(t *testing.T, normalized, output *schema.Transcript) {
|
||||
t.Helper()
|
||||
if len(normalized.Segments) != len(output.Segments) {
|
||||
t.Fatalf("segment membership changed: normalized=%+v output=%+v", normalized, output)
|
||||
}
|
||||
for i := range normalized.Segments {
|
||||
normalized.Segments[i].Text = output.Segments[i].Text
|
||||
}
|
||||
if !reflect.DeepEqual(normalized, output) {
|
||||
t.Fatalf("segment metadata/order changed: normalized=%+v output=%+v", normalized, output)
|
||||
}
|
||||
}
|
||||
|
||||
func ledgerDisposition(entry map[string]json.RawMessage) string {
|
||||
var value string
|
||||
_ = json.Unmarshal(entry["disposition"], &value)
|
||||
return value
|
||||
}
|
||||
|
||||
func ledgerHasDecision(t *testing.T, entry map[string]json.RawMessage, field, key string, approved bool) bool {
|
||||
t.Helper()
|
||||
var decisions []struct {
|
||||
ValidatorKey string `json:"validator_key"`
|
||||
Approved bool `json:"approved"`
|
||||
}
|
||||
if err := json.Unmarshal(entry[field], &decisions); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, decision := range decisions {
|
||||
if decision.ValidatorKey == key && decision.Approved == approved {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (f processFixture) assertNoStructuralRepairs(t *testing.T, wantCalls int) {
|
||||
t.Helper()
|
||||
dirs, err := filepath.Glob(filepath.Join(f.dir, "runs", "run-*"))
|
||||
if err != nil || len(dirs) != 1 {
|
||||
t.Fatalf("debug run: %v %v", dirs, err)
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(dirs[0], "llm-execution.json"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var execution struct {
|
||||
Calls []struct {
|
||||
Summary struct {
|
||||
ProviderAttempts int `json:"provider_attempts"`
|
||||
StructuralRepairs int `json:"structural_repairs"`
|
||||
} `json:"summary"`
|
||||
} `json:"calls"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &execution); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(execution.Calls) != wantCalls || f.provider.callCount() != wantCalls {
|
||||
t.Fatalf("unexpected generation counts: calls=%d provider=%d", len(execution.Calls), f.provider.callCount())
|
||||
}
|
||||
for _, call := range execution.Calls {
|
||||
if call.Summary.ProviderAttempts != 1 || call.Summary.StructuralRepairs != 0 {
|
||||
t.Fatalf("unexpected structural repair: %+v", call)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubprocessAppliesSpanDeletionAndRepetitionCleanup(t *testing.T) {
|
||||
f := newProcessFixture(t, "span-clean", "[spoken_word]")
|
||||
f.setSegments(t, "uh hello", "I I think")
|
||||
code, stdout, stderr := f.run(t, true, "--debug")
|
||||
if code != 0 || stdout != "" || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 0 recoveries") {
|
||||
t.Fatalf("code=%d stdout=%q stderr=%q", code, stdout, stderr)
|
||||
}
|
||||
output := f.readTranscript(t, f.output)
|
||||
normalized, ledger := f.debugArtifacts(t)
|
||||
assertSpanMetadata(t, normalized, output)
|
||||
if len(output.Segments) != 2 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" {
|
||||
t.Fatalf("unexpected corrected transcript: %+v", output)
|
||||
}
|
||||
if len(ledger) != 2 || ledgerDisposition(ledger[0]) != "applied" || ledgerDisposition(ledger[1]) != "applied" || string(ledger[0]["proposed_corrected_text"]) != `""` || string(ledger[0]["applied_corrected_text"]) != `""` {
|
||||
t.Fatalf("deletion missing from ledger: %+v", ledger)
|
||||
}
|
||||
errors, recoveries := f.errors(t), f.recoveries(t)
|
||||
if len(errors.Errors) != 0 || len(recoveries.Recoveries) != 0 || *errors.Summary.CorrectionsApplied != 2 || *errors.Summary.WorkUnitsCompleted != 1 || *recoveries.Summary.CorrectionsApplied != 2 {
|
||||
t.Fatalf("report totals disagree: errors=%+v recoveries=%+v", errors, recoveries)
|
||||
}
|
||||
f.assertNoStructuralRepairs(t, 3)
|
||||
}
|
||||
|
||||
func TestSubprocessAppliesWhitespaceOnlySpan(t *testing.T) {
|
||||
f := newProcessFixture(t, "span-spacing", "[grammar]")
|
||||
f.setSegments(t, "hello world")
|
||||
code, _, stderr := f.run(t, true, "--debug")
|
||||
if code != 0 || !strings.Contains(stderr, "1 corrections applied; 0 work units with errors; 0 recoveries") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
output := f.readTranscript(t, f.output)
|
||||
normalized, _ := f.debugArtifacts(t)
|
||||
assertSpanMetadata(t, normalized, output)
|
||||
if len(output.Segments) != 1 || output.Segments[0].Text != "hello world" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
|
||||
t.Fatalf("spacing correction failed: %+v", output)
|
||||
}
|
||||
f.assertNoStructuralRepairs(t, 3)
|
||||
}
|
||||
|
||||
func TestSubprocessFiltersBlankResultAlongsideApprovedCorrections(t *testing.T) {
|
||||
f := newProcessFixture(t, "span-filter", "[spoken_word]")
|
||||
f.setSegments(t, "uh hello", "I I think", "uh")
|
||||
code, _, stderr := f.run(t, true, "--debug")
|
||||
if code != 0 || !strings.Contains(stderr, "2 corrections applied; 0 work units with errors; 1 recoveries") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
output := f.readTranscript(t, f.output)
|
||||
normalized, ledger := f.debugArtifacts(t)
|
||||
assertSpanMetadata(t, normalized, output)
|
||||
if len(output.Segments) != 3 || output.Segments[0].Text != "hello" || output.Segments[1].Text != "I think" || output.Segments[2].Text != "uh" {
|
||||
t.Fatalf("filtered segment or survivors wrong: %+v", output)
|
||||
}
|
||||
recoveries := f.recoveries(t)
|
||||
if len(f.errors(t).Errors) != 0 || len(recoveries.Recoveries) != 1 || recoveries.Recoveries[0].Operation != "proposal" || len(recoveries.Recoveries[0].Reasons) != 1 || recoveries.Recoveries[0].Reasons[0] != "proposals_filtered" || len(recoveries.Recoveries[0].FilteredProposals) != 1 || recoveries.Recoveries[0].FilteredProposals[0].Reason != "empty_resulting_segment" || *recoveries.Summary.ProposalResponsesWithFiltering != 1 {
|
||||
t.Fatalf("filtering recovery wrong: %+v", recoveries)
|
||||
}
|
||||
if len(ledger) != 3 || ledgerDisposition(ledger[2]) != "rejected" || string(ledger[2]["proposed_corrected_text"]) != `""` {
|
||||
t.Fatalf("filtered deletion evidence missing: %+v", ledger)
|
||||
}
|
||||
if !ledgerHasDecision(t, ledger[2], "deterministic_validator_decisions", "non_empty_resulting_segment", false) {
|
||||
t.Fatalf("renamed validator missing from evidence: %+v", ledger[2])
|
||||
}
|
||||
f.assertNoStructuralRepairs(t, 3)
|
||||
}
|
||||
|
||||
func TestSubprocessRespectsNegativeMeaningReviewOnDeletion(t *testing.T) {
|
||||
f := newProcessFixture(t, "span-negative", "[grammar]")
|
||||
f.setSegments(t, "not good")
|
||||
code, _, stderr := f.run(t, true, "--debug")
|
||||
if code != 0 || !strings.Contains(stderr, "0 corrections applied; 0 work units with errors; 0 recoveries") {
|
||||
t.Fatalf("code=%d stderr=%q", code, stderr)
|
||||
}
|
||||
output := f.readTranscript(t, f.output)
|
||||
normalized, ledger := f.debugArtifacts(t)
|
||||
assertSpanMetadata(t, normalized, output)
|
||||
if len(output.Segments) != 1 || output.Segments[0].Text != "not good" || len(f.errors(t).Errors) != 0 || len(f.recoveries(t).Recoveries) != 0 {
|
||||
t.Fatalf("negative review changed outcome: %+v", output)
|
||||
}
|
||||
if len(ledger) != 1 || ledgerDisposition(ledger[0]) != "rejected" || string(ledger[0]["proposed_corrected_text"]) != `""` {
|
||||
t.Fatalf("rejection evidence missing: %+v", ledger)
|
||||
}
|
||||
if _, present := ledger[0]["applied_corrected_text"]; present || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "editorial_review", true) || !ledgerHasDecision(t, ledger[0], "llm_validator_decisions", "meaning_reversal_review", false) {
|
||||
t.Fatalf("negative decision not recorded: %+v", ledger[0])
|
||||
}
|
||||
f.assertNoStructuralRepairs(t, 3)
|
||||
}
|
||||
|
||||
func TestSubprocessDefaultFileOutputAndReports(t *testing.T) {
|
||||
f := newProcessFixture(t, "normal", "[grammar]")
|
||||
code, stdout, stderr := f.run(t, true)
|
||||
|
||||
Reference in New Issue
Block a user