eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 12 grammar module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 02:57:06 +00:00
parent b360493cdc
commit fc3a7b7a67
12 changed files with 816 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
internal/cli/run.go
View File

@@ -18,15 +18,18 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
type processInvocation struct {
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
ExplicitModules bool
}
var processModuleFactory runner.ModuleFactory
@@ -127,22 +130,71 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
workingTranscript := normalizedTranscript
var runOutput *runner.RunOutput
if processModuleFactory != nil {
moduleFactory := processModuleFactory
if moduleFactory == nil && inv.ExplicitModules {
moduleFactory = modules.NewFactory(modules.Dependencies{
Config: &inv.Config,
Glossary: glossary,
DiagnosticsDir: runDir.Path(),
})
}
if moduleFactory != nil {
proposalLLMClient := processProposalLLMClient
validationLLMClient := processValidationLLMClient
proposalScheduler := processProposalLLMScheduler
validationScheduler := processValidationLLMScheduler
if processModuleFactory == nil {
// Production runtime path: construct clients/schedulers from config.
if proposalLLMClient == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
proposalLLMClient = client
}
if validationLLMClient == nil {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
validationLLMClient = client
}
if proposalScheduler == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
s, sErr := llm.NewScheduler(primaryCfg.Concurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
proposalScheduler = s
}
if validationScheduler == nil {
validationCfg := llm.ResolveValidationConfig(inv.Config)
s, sErr := llm.NewScheduler(validationCfg.Concurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
validationScheduler = s
}
}
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
if err != nil {
return fail("runner_setup", err, nil)
}
runnerResult, runErr := runner.New(processModuleFactory).Run(context.Background(), runner.RunInput{
runnerResult, runErr := runner.New(moduleFactory).Run(context.Background(), runner.RunInput{
Config: &inv.Config,
Transcript: normalizedTranscript,
Glossary: glossary,
ModuleSpecs: moduleSpecs,
ProposalLLMClient: processProposalLLMClient,
ProposalLLMScheduler: processProposalLLMScheduler,
ProposalLLMClient: proposalLLMClient,
ProposalLLMScheduler: proposalScheduler,
ProposalDiagnosticsDir: runDir.Path(),
ValidationLLMClient: processValidationLLMClient,
ValidationLLMScheduler: processValidationLLMScheduler,
ValidationLLMClient: validationLLMClient,
ValidationLLMScheduler: validationScheduler,
ValidationDiagnosticsDir: runDir.Path(),
})
runOutput = &runnerResult
@@ -243,9 +295,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
overrides := config.CLIOverrides{}
explicitModules := false
fs.Visit(func(f *flag.Flag) {
switch f.Name {
case "modules":
explicitModules = true
overrides.ModulesCSV = pFlags.modules
case "llm-api-key":
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
@@ -322,11 +376,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
inv := processInvocation{
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
ExplicitModules: explicitModules,
}
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
@@ -405,7 +460,7 @@ func extractErrorPhase(err error) (phase string, message string) {
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase11-proposal-generation-framework",
Phase: "phase12-grammar-module",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,

278
internal/cli/run_test.go
View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -615,8 +616,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase11-proposal-generation-framework" {
t.Errorf("expected phase 'phase11-proposal-generation-framework', got %q", report.Phase)
if report.Phase != "phase12-grammar-module" {
t.Errorf("expected phase 'phase12-grammar-module', got %q", report.Phase)
}
}
@@ -661,8 +662,9 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq
}
type fakeStructuredLLMClient struct {
responses []validators.LLMValidationResponse
err error
validationResponses []validators.LLMValidationResponse
proposalResponses []proposal_generation.StructuredCorrectionSet
err error
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -671,13 +673,24 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm call")
switch target := out.(type) {
case *validators.LLMValidationResponse:
if len(f.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = f.validationResponses[0]
f.validationResponses = f.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *proposal_generation.StructuredCorrectionSet:
if len(f.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = f.proposalResponses[0]
f.proposalResponses = f.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
target := out.(*validators.LLMValidationResponse)
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
@@ -771,7 +784,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
processValidationLLMClient = &fakeStructuredLLMClient{
responses: []validators.LLMValidationResponse{{
validationResponses: []validators.LLMValidationResponse{{
Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject"}},
}},
}
@@ -940,6 +953,249 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
}
}
func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word"} {
t.Run(moduleKey, func(t *testing.T) {
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", moduleKey,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected failure for %s", moduleKey)
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "recognized but not implemented") {
t.Fatalf("expected unimplemented message, got %q", stderr.String())
}
})
}
}
func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *testing.T) {
secret := "phase12-secret"
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
},
}
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Hello, world" {
t.Fatalf("expected grammar correction applied, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 || report.ModuleResults[0].ModuleKey != "grammar" {
t.Fatalf("expected one grammar module result, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied grammar change, got %+v", report.ModuleResults[0].AppliedChanges)
}
if len(report.ModuleResults[0].ValidatorDecisions) == 0 {
t.Fatalf("expected validator decisions in report")
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 1 {
t.Fatalf("expected module results in run-dir report")
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "grammar", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected grammar diagnostics payload files in %s", filepath.Join(runDir, "grammar"))
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "Hello world", CorrectedText: "Hi world", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "world", CorrectedText: "earth", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.9, Reason: "reject"},
},
},
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result")
}
module := report.ModuleResults[0]
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if len(module.SkippedChanges) != 1 {
t.Fatalf("expected one application skip, got %+v", module.SkippedChanges)
}
if module.ValidatorRejected[0].ReasonCode == string(module.SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
}
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed grammar run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
}
func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *testing.T) {
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ModulesSummary != nil || len(report.ModuleResults) != 0 {
t.Fatalf("expected no module execution without explicit --modules, got summary=%+v results=%+v", report.ModulesSummary, report.ModuleResults)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer