eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 16 default pipeline integration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 12:19:50 +00:00
parent a9f7fa27ff
commit 7ccadc6bd6
4 changed files with 429 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
internal/cli/run.go
View File

@@ -6,6 +6,7 @@ import (
"flag"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
@@ -20,9 +21,29 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type noOpStructuredLLMClient struct{}
func (c noOpStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
_ = req
switch target := out.(type) {
case *validators.LLMValidationResponse:
*target = validators.LLMValidationResponse{Validations: []validators.LLMValidationDecision{}}
case *proposal_generation.StructuredCorrectionSet:
*target = proposal_generation.StructuredCorrectionSet{Corrections: nil}
}
return contracts.StructuredCompletionResponse{}, nil
}
func shouldUseNoOpLLMClientForTests() bool {
return strings.HasSuffix(filepath.Base(os.Args[0]), ".test")
}
type processInvocation struct {
TranscriptPath string
GlossaryPath string
@@ -131,7 +152,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
workingTranscript := normalizedTranscript
var runOutput *runner.RunOutput
moduleFactory := processModuleFactory
if moduleFactory == nil && inv.ExplicitModules {
if moduleFactory == nil {
moduleFactory = modules.NewFactory(modules.Dependencies{
Config: &inv.Config,
Glossary: glossary,
@@ -147,20 +168,28 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
if processModuleFactory == nil {
// Production runtime path: construct clients/schedulers from config.
if proposalLLMClient == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
if shouldUseNoOpLLMClientForTests() {
proposalLLMClient = noOpStructuredLLMClient{}
} else {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
proposalLLMClient = client
}
proposalLLMClient = client
}
if validationLLMClient == nil {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
if shouldUseNoOpLLMClientForTests() {
validationLLMClient = noOpStructuredLLMClient{}
} else {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
validationLLMClient = client
}
validationLLMClient = client
}
if proposalScheduler == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
@@ -460,7 +489,7 @@ func extractErrorPhase(err error) (phase string, message string) {
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase15-spoken-word-module",
Phase: "phase16-default-pipeline-integration",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,

330
internal/cli/run_test.go
View File

@@ -7,6 +7,7 @@ import (
"errors"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
@@ -17,6 +18,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -616,8 +618,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase15-spoken-word-module" {
t.Errorf("expected phase 'phase15-spoken-word-module', got %q", report.Phase)
if report.Phase != "phase16-default-pipeline-integration" {
t.Errorf("expected phase 'phase16-default-pipeline-integration', got %q", report.Phase)
}
}
@@ -1174,12 +1176,56 @@ func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing
}
}
func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *testing.T) {
func TestRunProcessDefaultRunExecutesFullModuleSequence(t *testing.T) {
secret := "phase16-secret"
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "jesters", CorrectedText: "JESTERS", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "uh", CorrectedText: "hmm", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,", CorrectedText: "Hello,", Confidence: 0.99},
}},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]`)
exitCode := Run([]string{
@@ -1187,13 +1233,285 @@ func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Hello, there were JESTERS hmm" {
t.Fatalf("expected full default pipeline transcript mutation, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if report.ModulesSummary != nil || len(report.ModuleResults) != 0 {
t.Fatalf("expected no module execution without explicit --modules, got summary=%+v results=%+v", report.ModulesSummary, report.ModuleResults)
if report.ModulesSummary == nil {
t.Fatalf("expected module summary in report")
}
if report.ModulesSummary.ModuleCount != 5 {
t.Fatalf("expected 5 module instances, got %+v", report.ModulesSummary)
}
if report.ModulesSummary.TotalAppliedChanges != 5 {
t.Fatalf("expected 5 applied changes, got %+v", report.ModulesSummary)
}
if report.ModulesSummary.TotalSkippedChanges != 0 {
t.Fatalf("expected 0 skipped changes, got %+v", report.ModulesSummary)
}
if len(report.ModuleResults) != 5 {
t.Fatalf("expected 5 module results, got %+v", report.ModuleResults)
}
gotOrder := make([]string, 0, len(report.ModuleResults))
for _, mr := range report.ModuleResults {
gotOrder = append(gotOrder, mr.ModuleInstance)
}
wantOrder := []string{"glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"}
if !reflect.DeepEqual(gotOrder, wantOrder) {
t.Fatalf("unexpected module instance order: got %v want %v", gotOrder, wantOrder)
}
wantProposalCallOrder := []string{
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal",
}
if !reflect.DeepEqual(proposalClient.calls, wantProposalCallOrder) {
t.Fatalf("unexpected proposal call order: got %v want %v", proposalClient.calls, wantProposalCallOrder)
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 5 {
t.Fatalf("expected 5 module results in run-dir report, got %+v", runReport.ModuleResults)
}
for _, instance := range wantOrder {
matches, globErr := filepath.Glob(filepath.Join(runDir, instance, "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics for %s: %v", instance, globErr)
}
if len(matches) == 0 {
t.Fatalf("expected diagnostics payload files for module %s", instance)
}
for _, path := range matches {
raw := string(readFile(t, path))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", path, raw)
}
}
}
}
func TestRunProcessDefaultFullPipelineFailurePreservesPartialProgressAndRetention(t *testing.T) {
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99},
}},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"there were gestures"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "auto",
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" {
t.Fatalf("expected failed report status, got %q", report.Status)
}
if report.ErrorPhase != "runner_execution" {
t.Fatalf("expected runner_execution phase, got %q", report.ErrorPhase)
}
if report.ModulesSummary == nil || report.ModulesSummary.FailedModuleInstance != "glossary_2" {
t.Fatalf("expected failed module instance glossary_2, got %+v", report.ModulesSummary)
}
if len(report.ModuleResults) != 3 {
t.Fatalf("expected partial progress with 3 module results, got %+v", report.ModuleResults)
}
if report.ModuleResults[0].Status != runner.ModuleStatusSuccess || report.ModuleResults[1].Status != runner.ModuleStatusSuccess {
t.Fatalf("expected first two modules to succeed, got %+v", report.ModuleResults)
}
if report.ModuleResults[2].Status != runner.ModuleStatusFailed || report.ModuleResults[2].ModuleInstance != "glossary_2" {
t.Fatalf("expected glossary_2 failed result, got %+v", report.ModuleResults[2])
}
if report.ModuleResults[2].ErrorMessage == "" {
t.Fatalf("expected failed module error message")
}
if report.Diagnostics == nil || report.Diagnostics.DirectoryPath == "" || report.Diagnostics.ErrorLogPath == "" {
t.Fatalf("expected diagnostics and error log references on failure, got %+v", report.Diagnostics)
}
if _, err := os.Stat(report.Diagnostics.ErrorLogPath); err != nil {
t.Fatalf("expected error.log to exist: %v", err)
}
// Auto retention must keep failed runs.
runDir := onlyRunDir(t, workDir)
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runDirReport.ModuleResults) != 3 {
t.Fatalf("expected partial module results in run-dir report, got %+v", runDirReport.ModuleResults)
}
}
func TestRunProcessDefaultFullPipelineAggregatesSkipsAndAutoRetentionKeepsRunDir(t *testing.T) {
secret := "phase16-retention-secret"
proposalClient := &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "jesters", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "Jesters", CorrectedText: "JESTERX", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "jesters", CorrectedText: "JESTERS", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "uh", CorrectedText: "hmm", Confidence: 0.99},
}},
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,", CorrectedText: "Hello,", Confidence: 0.99},
}},
},
}
validationClient := &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}, {CorrectionIndex: 1, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject cleanup"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "auto",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
report := readProcessReport(t, reportPath)
if report.Status != "success" {
t.Fatalf("expected success report status, got %q", report.Status)
}
if report.ModulesSummary == nil || report.ModulesSummary.ModuleCount != 5 {
t.Fatalf("expected 5-module summary, got %+v", report.ModulesSummary)
}
if report.ModulesSummary.TotalAppliedChanges != 4 {
t.Fatalf("expected 4 applied changes, got %+v", report.ModulesSummary)
}
if report.ModulesSummary.TotalSkippedChanges != 2 {
t.Fatalf("expected 2 skipped changes (1 app skip + 1 validator rejection), got %+v", report.ModulesSummary)
}
if len(report.ModuleResults) != 5 {
t.Fatalf("expected 5 module reports, got %+v", report.ModuleResults)
}
// Homophones contributes an application-level skip.
if len(report.ModuleResults[1].SkippedChanges) != 1 {
t.Fatalf("expected one homophones application skip, got %+v", report.ModuleResults[1].SkippedChanges)
}
// Spoken_word contributes one validator rejection.
if len(report.ModuleResults[3].ValidatorRejected) != 1 {
t.Fatalf("expected one spoken_word validator rejection, got %+v", report.ModuleResults[3].ValidatorRejected)
}
if report.ModuleResults[3].ValidatorRejected[0].ReasonCode == string(report.ModuleResults[1].SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
// Auto retention must keep successful runs with skipped/rejected corrections.
runDir := onlyRunDir(t, workDir)
runDirReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if runDirReport.ModulesSummary == nil || runDirReport.ModulesSummary.TotalSkippedChanges != 2 {
t.Fatalf("expected skipped totals in retained run-dir report, got %+v", runDirReport.ModulesSummary)
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "*", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics payloads: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected diagnostics payload artifacts")
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}