Complete Phase 12 grammar module

This commit is contained in:
2026-05-12 02:57:06 +00:00
parent b360493cdc
commit fc3a7b7a67
12 changed files with 816 additions and 88 deletions

View File

@@ -18,15 +18,18 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/runner"
)
type processInvocation struct {
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
TranscriptPath string
GlossaryPath string
OutputPath string
ReportJSONPath string
Config config.Config
ExplicitModules bool
}
var processModuleFactory runner.ModuleFactory
@@ -127,22 +130,71 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
workingTranscript := normalizedTranscript
var runOutput *runner.RunOutput
if processModuleFactory != nil {
moduleFactory := processModuleFactory
if moduleFactory == nil && inv.ExplicitModules {
moduleFactory = modules.NewFactory(modules.Dependencies{
Config: &inv.Config,
Glossary: glossary,
DiagnosticsDir: runDir.Path(),
})
}
if moduleFactory != nil {
proposalLLMClient := processProposalLLMClient
validationLLMClient := processValidationLLMClient
proposalScheduler := processProposalLLMScheduler
validationScheduler := processValidationLLMScheduler
if processModuleFactory == nil {
// Production runtime path: construct clients/schedulers from config.
if proposalLLMClient == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(primaryCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
proposalLLMClient = client
}
if validationLLMClient == nil {
validationCfg := llm.ResolveValidationConfig(inv.Config)
client, clientErr := llm.NewInstructorClient(validationCfg.ToInstructorClientConfig(llm.ModeJSON, nil))
if clientErr != nil {
return fail("runner_setup", clientErr, nil)
}
validationLLMClient = client
}
if proposalScheduler == nil {
primaryCfg := llm.ResolvePrimaryConfig(inv.Config)
s, sErr := llm.NewScheduler(primaryCfg.Concurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
proposalScheduler = s
}
if validationScheduler == nil {
validationCfg := llm.ResolveValidationConfig(inv.Config)
s, sErr := llm.NewScheduler(validationCfg.Concurrency)
if sErr != nil {
return fail("runner_setup", sErr, nil)
}
validationScheduler = s
}
}
moduleSpecs, err := contracts.ResolveModuleRunSpecs(inv.Config.Modules)
if err != nil {
return fail("runner_setup", err, nil)
}
runnerResult, runErr := runner.New(processModuleFactory).Run(context.Background(), runner.RunInput{
runnerResult, runErr := runner.New(moduleFactory).Run(context.Background(), runner.RunInput{
Config: &inv.Config,
Transcript: normalizedTranscript,
Glossary: glossary,
ModuleSpecs: moduleSpecs,
ProposalLLMClient: processProposalLLMClient,
ProposalLLMScheduler: processProposalLLMScheduler,
ProposalLLMClient: proposalLLMClient,
ProposalLLMScheduler: proposalScheduler,
ProposalDiagnosticsDir: runDir.Path(),
ValidationLLMClient: processValidationLLMClient,
ValidationLLMScheduler: processValidationLLMScheduler,
ValidationLLMClient: validationLLMClient,
ValidationLLMScheduler: validationScheduler,
ValidationDiagnosticsDir: runDir.Path(),
})
runOutput = &runnerResult
@@ -243,9 +295,11 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
overrides := config.CLIOverrides{}
explicitModules := false
fs.Visit(func(f *flag.Flag) {
switch f.Name {
case "modules":
explicitModules = true
overrides.ModulesCSV = pFlags.modules
case "llm-api-key":
overrides.PrimaryLLMAPIKey = pFlags.llmAPIKey
@@ -322,11 +376,12 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
}
inv := processInvocation{
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
TranscriptPath: positional[0],
GlossaryPath: *pFlags.glossaryPath,
OutputPath: *pFlags.outputPath,
ReportJSONPath: *pFlags.reportJSONPath,
Config: cfg,
ExplicitModules: explicitModules,
}
normSummary, chunkSummary, runOutput, runDir, runErr := processRunner(inv, stdout)
@@ -405,7 +460,7 @@ func extractErrorPhase(err error) (phase string, message string) {
func buildProcessReport(status string, inv processInvocation, runDir *diagnostics.RunDirectory, startedAt, completedAt time.Time, errorMessage string, errorPhase string, normalizationSummary *normalization.NormalizationSummary, chunkingSummary *chunking.Summary, runOutput *runner.RunOutput) reporting.ProcessReport {
report := reporting.ProcessReport{
Phase: "phase11-proposal-generation-framework",
Phase: "phase12-grammar-module",
Status: status,
Operation: "process",
TranscriptPath: inv.TranscriptPath,

View File

@@ -15,6 +15,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
@@ -615,8 +616,8 @@ func TestRunProcessReportJSONIncludesChunkingSummary(t *testing.T) {
if report.Chunking.MaxSectionTokens == 0 {
t.Errorf("expected max_section_tokens in report")
}
if report.Phase != "phase11-proposal-generation-framework" {
t.Errorf("expected phase 'phase11-proposal-generation-framework', got %q", report.Phase)
if report.Phase != "phase12-grammar-module" {
t.Errorf("expected phase 'phase12-grammar-module', got %q", report.Phase)
}
}
@@ -661,8 +662,9 @@ func (v fakeValidator) Validate(ctx context.Context, req contracts.ValidationReq
}
type fakeStructuredLLMClient struct {
responses []validators.LLMValidationResponse
err error
validationResponses []validators.LLMValidationResponse
proposalResponses []proposal_generation.StructuredCorrectionSet
err error
}
func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
@@ -671,13 +673,24 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm call")
switch target := out.(type) {
case *validators.LLMValidationResponse:
if len(f.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = f.validationResponses[0]
f.validationResponses = f.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *proposal_generation.StructuredCorrectionSet:
if len(f.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = f.proposalResponses[0]
f.proposalResponses = f.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
target := out.(*validators.LLMValidationResponse)
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
@@ -771,7 +784,7 @@ func TestRunProcessInjectedFactoryLLMValidatorResultsInReports(t *testing.T) {
t.Fatalf("NewLLMBackedValidator: %v", err)
}
processValidationLLMClient = &fakeStructuredLLMClient{
responses: []validators.LLMValidationResponse{{
validationResponses: []validators.LLMValidationResponse{{
Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: false, Confidence: 0.99, Reason: "reject"}},
}},
}
@@ -940,6 +953,249 @@ func TestRunProcessProductionRegistryUnimplementedModuleFailsCleanly(t *testing.
}
}
func TestRunProcessExplicitUnimplementedModulesFailClearly(t *testing.T) {
for _, moduleKey := range []string{"glossary", "homophones", "spoken_word"} {
t.Run(moduleKey, func(t *testing.T) {
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", moduleKey,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected failure for %s", moduleKey)
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "recognized but not implemented") {
t.Fatalf("expected unimplemented message, got %q", stderr.String())
}
})
}
}
func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *testing.T) {
secret := "phase12-secret"
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: secret}}},
},
}
t.Setenv("AUDITA_LLM_API_KEY", secret)
t.Setenv("AUDITA_VALIDATION_LLM_API_KEY", secret)
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
"--work-dir", workDir,
"--work-dir-retention", "always",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout with --output, got %q", stdout.String())
}
parsed, err := schema.ParseTranscriptJSON(readFile(t, outputPath))
if err != nil {
t.Fatalf("parse output: %v", err)
}
if parsed.Segments[0].Text != "Hello, world" {
t.Fatalf("expected grammar correction applied, got %q", parsed.Segments[0].Text)
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 || report.ModuleResults[0].ModuleKey != "grammar" {
t.Fatalf("expected one grammar module result, got %+v", report.ModuleResults)
}
if len(report.ModuleResults[0].AppliedChanges) != 1 {
t.Fatalf("expected one applied grammar change, got %+v", report.ModuleResults[0].AppliedChanges)
}
if len(report.ModuleResults[0].ValidatorDecisions) == 0 {
t.Fatalf("expected validator decisions in report")
}
runDir := onlyRunDir(t, workDir)
runReport := readProcessReport(t, filepath.Join(runDir, "report.json"))
if len(runReport.ModuleResults) != 1 {
t.Fatalf("expected module results in run-dir report")
}
diagFiles, globErr := filepath.Glob(filepath.Join(runDir, "grammar", "*response-payload.json"))
if globErr != nil {
t.Fatalf("glob diagnostics: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected grammar diagnostics payload files in %s", filepath.Join(runDir, "grammar"))
}
for _, f := range diagFiles {
raw := string(readFile(t, f))
if strings.Contains(raw, secret) {
t.Fatalf("secret leaked in diagnostics %q: %s", f, raw)
}
}
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "Hello", CorrectedText: "Hi", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "Hello world", CorrectedText: "Hi world", Confidence: 0.99},
{TargetSegmentID: 1, OriginalText: "world", CorrectedText: "earth", Confidence: 0.99},
},
},
},
}
processValidationLLMClient = &fakeStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 2, Approved: false, Confidence: 0.9, Reason: "reject"},
},
},
{
Validations: []validators.LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},
{CorrectionIndex: 1, Approved: true, Confidence: 0.9, Reason: "ok"},
},
},
},
}
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
})
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if len(report.ModuleResults) != 1 {
t.Fatalf("expected one module result")
}
module := report.ModuleResults[0]
if len(module.ValidatorRejected) != 1 {
t.Fatalf("expected one validator rejection, got %+v", module.ValidatorRejected)
}
if len(module.SkippedChanges) != 1 {
t.Fatalf("expected one application skip, got %+v", module.SkippedChanges)
}
if module.ValidatorRejected[0].ReasonCode == string(module.SkippedChanges[0].SkipReason) {
t.Fatalf("validator rejection and application skip should remain distinct")
}
}
func TestRunProcessExplicitGrammarMalformedLLMOutputFailsWithErrorLog(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{err: errors.New("malformed structured output")}
t.Cleanup(func() { processProposalLLMClient = nil })
var stdout, stderr bytes.Buffer
workDir := t.TempDir()
reportPath := filepath.Join(t.TempDir(), "report.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--work-dir", workDir,
"--work-dir-retention", "always",
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatal("expected failure")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout on failure, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "runner_execution") {
t.Fatalf("expected runner_execution error, got %q", stderr.String())
}
runDir := onlyRunDir(t, workDir)
if _, err := os.Stat(filepath.Join(runDir, "error.log")); err != nil {
t.Fatalf("expected error.log on failed grammar run: %v", err)
}
report := readProcessReport(t, reportPath)
if report.Status != "failed" || report.ErrorPhase != "runner_execution" {
t.Fatalf("expected failed runner_execution report, got %+v", report)
}
}
func TestRunProcessDefaultBehaviorRemainsDeterministicWithoutExplicitModules(t *testing.T) {
var stdout, stderr bytes.Buffer
reportPath := filepath.Join(t.TempDir(), "report.json")
outputPath := filepath.Join(t.TempDir(), "out.json")
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--output", outputPath,
"--report-json", reportPath,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
report := readProcessReport(t, reportPath)
if report.ModulesSummary != nil || len(report.ModuleResults) != 0 {
t.Fatalf("expected no module execution without explicit --modules, got summary=%+v results=%+v", report.ModulesSummary, report.ModuleResults)
}
}
func TestRunProcessChunkingSummaryArtifactWritten(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer

View File

@@ -11,7 +11,7 @@ import (
func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
now := time.Now().UTC()
report := ProcessReport{
Phase: "phase11-proposal-generation-framework",
Phase: "phase12-grammar-module",
Status: "success",
ModuleResults: []ModuleReport{
{
@@ -80,7 +80,7 @@ func TestProcessReportModuleResultsJSONSuccessAndSkipped(t *testing.T) {
func TestProcessReportModuleResultsJSONFailedModule(t *testing.T) {
now := time.Now().UTC()
report := ProcessReport{
Phase: "phase11-proposal-generation-framework",
Phase: "phase12-grammar-module",
Status: "failed",
ModuleResults: []ModuleReport{
{

View File

@@ -8,6 +8,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
grammarmodule "gitea.maximumdirect.net/eric/audita/internal/modules/grammar"
)
const (
@@ -65,10 +66,12 @@ type Factory struct {
// NewFactory creates a production registry scaffold with known module keys but
// no real module constructors registered yet.
func NewFactory(deps Dependencies) *Factory {
return &Factory{
factory := &Factory{
deps: deps,
constructors: make(map[string]Constructor, len(knownModuleKeys)),
}
_ = factory.RegisterConstructor(ModuleKeyGrammar, constructGrammarModule)
return factory
}
// RegisterConstructor registers a constructor for a known module key.
@@ -87,6 +90,12 @@ func (f *Factory) RegisterConstructor(moduleKey string, constructor Constructor)
return nil
}
func constructGrammarModule(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {
_ = ctx
_ = req
return grammarmodule.New()
}
// ModuleForSpec resolves one configured run spec into a module instance.
func (f *Factory) ModuleForSpec(spec contracts.ModuleRunSpec) (contracts.TranscriptModule, error) {
if f == nil {

View File

@@ -66,7 +66,7 @@ func TestUnsupportedUnknownModuleKeyFailsCleanly(t *testing.T) {
func TestRecognizedButUnimplementedModuleKeyFailsCleanly(t *testing.T) {
factory := NewFactory(Dependencies{})
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord, ModuleKeyGrammar} {
for _, key := range []string{ModuleKeyGlossary, ModuleKeyHomophones, ModuleKeySpokenWord} {
t.Run(key, func(t *testing.T) {
_, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: key, InstanceName: key})
if err == nil {
@@ -84,6 +84,17 @@ func TestRecognizedButUnimplementedModuleKeyFailsCleanly(t *testing.T) {
}
}
func TestGrammarIsRegisteredAndConstructibleByDefault(t *testing.T) {
factory := NewFactory(Dependencies{})
module, err := factory.ModuleForSpec(contracts.ModuleRunSpec{ModuleKey: ModuleKeyGrammar, InstanceName: ModuleKeyGrammar})
if err != nil {
t.Fatalf("ModuleForSpec error: %v", err)
}
if module.Key() != ModuleKeyGrammar {
t.Fatalf("expected grammar module key, got %q", module.Key())
}
}
func TestRegisterConstructorAndConstruct(t *testing.T) {
factory := NewFactory(Dependencies{})
if err := factory.RegisterConstructor(ModuleKeyGlossary, func(ctx context.Context, req ConstructRequest) (contracts.TranscriptModule, error) {

View File

@@ -54,6 +54,7 @@ type Request struct {
Config *config.Config `json:"-"`
Messages []contracts.LLMMessage `json:"messages"`
Model string `json:"model,omitempty"`
StageName string `json:"stage_name,omitempty"`
StartIndex int `json:"start_index"`
LLMClient contracts.StructuredLLMClient
Scheduler contracts.LLMScheduler
@@ -87,7 +88,10 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
return Result{}, fmt.Errorf("messages must not be empty")
}
stage := buildStageName(req.ModuleInstance, req.Section)
stage := strings.TrimSpace(req.StageName)
if stage == "" {
stage = buildStageName(req.ModuleInstance, req.Section)
}
model := resolveModel(req.Config, req.Model)
messages := append([]contracts.LLMMessage(nil), req.Messages...)

View File

@@ -12,6 +12,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
"gitea.maximumdirect.net/eric/audita/internal/framework/modules"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
@@ -651,3 +652,37 @@ func TestRunnerProposalGenerationHelperFlowsThroughPipeline(t *testing.T) {
t.Fatalf("expected proposal-generation diagnostics artifacts in %s", filepath.Join(diagDir, "m"))
}
}
func TestRunnerGrammarModuleUsesConfidenceThreshold(t *testing.T) {
client := &fakeProposalStructuredClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.5},
},
},
},
}
cfg := config.Default()
cfg.Thresholds.Grammar = 0.9
factory := modules.NewFactory(modules.Dependencies{})
out, err := New(factory).Run(context.Background(), RunInput{
Config: &cfg,
Transcript: &schema.Transcript{Segments: []schema.Segment{{ID: 1, Text: "hello ,world"}}},
Glossary: &schema.Glossary{},
ModuleSpecs: []contracts.ModuleRunSpec{{ModuleKey: "grammar", InstanceName: "grammar"}},
ProposalLLMClient: client,
})
if err != nil {
t.Fatalf("Run error: %v", err)
}
if out.FinalTranscript.Segments[0].Text != "hello ,world" {
t.Fatalf("expected no changes due to confidence threshold, got %q", out.FinalTranscript.Segments[0].Text)
}
if len(out.ModuleResults) != 1 || len(out.ModuleResults[0].ValidatorRejected) == 0 {
t.Fatalf("expected validator rejection, got %+v", out.ModuleResults)
}
if out.ModuleResults[0].ValidatorRejected[0].ReasonCode != validators.ReasonLowConfidence {
t.Fatalf("expected low confidence reason, got %+v", out.ModuleResults[0].ValidatorRejected[0])
}
}

View File

@@ -0,0 +1,76 @@
package grammar
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type Module struct {
validators []contracts.Validator
}
func New() (*Module, error) {
grammarOnlyGuard, err := validators.NewLLMBackedValidator("grammar_only_guard", validators.LLMValidatorTypeGrammarReview, "")
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
grammarOnlyGuard,
meaningReversal,
},
}, nil
}
func (m *Module) Key() string { return "grammar" }
func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
// Python grammar module uses require_unique for conservative single-span replacement.
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
messages, err := BuildProposalMessages(req.WorkingTranscript, req.Glossary)
if err != nil {
return nil, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
StageName: fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
}
return generated.Corrections, nil
}

View File

@@ -0,0 +1,194 @@
package grammar
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
)
type fakeLLMClient struct {
responses []map[string]any
err error
calls []contracts.StructuredCompletionRequest
}
func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
payload := f.responses[0]
f.responses = f.responses[1:]
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
items, _ := payload["corrections"].([]map[string]any)
for _, item := range items {
target.Corrections = append(target.Corrections, proposal_generation.StructuredCorrectionProposal{
TargetSegmentID: item["id"].(int),
OriginalText: item["original_text"].(string),
CorrectedText: item["corrected_text"].(string),
Confidence: item["confidence"].(float64),
})
}
return contracts.StructuredCompletionResponse{}, nil
}
type countingScheduler struct {
runs int
}
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
s.runs++
return fn(ctx)
}
func tinyTranscript() *schema.Transcript {
return &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "hello ,world"},
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction", Aliases: []string{"Jester"}},
}}
}
func TestBuildProposalMessagesConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary())
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
if len(msgs) != 2 {
t.Fatalf("expected 2 messages, got %d", len(msgs))
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, want := range []string{
"Transcript section:",
"Protected glossary/context:",
"punctuation, capitalization, spacing, and article cleanup only",
"Do not make word substitutions",
"Do not return speaker, start, or end fields",
`"id": 1`,
} {
if !strings.Contains(combined, want) {
t.Fatalf("expected prompt to contain %q", want)
}
}
if strings.Contains(strings.ToLower(combined), "summarize") {
t.Fatalf("prompt should not invite summarization")
}
}
func TestGrammarModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique {
t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy())
}
}
func TestGrammarModuleValidatorChain(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
got := make([]string, 0, len(m.Validators()))
for _, v := range m.Validators() {
got = append(got, v.Name())
}
want := []string{
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"grammar_only_guard",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want)
}
}
func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
secret := "super-secret-key"
client := &fakeLLMClient{
responses: []map[string]any{
{
"corrections": []map[string]any{
{"id": 1, "original_text": "hello ,world", "corrected_text": "Hello, world", "confidence": 0.93},
},
},
},
}
scheduler := &countingScheduler{}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
diagDir := t.TempDir()
out, err := module.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "grammar",
InstanceName: "grammar",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
},
LLMClient: client,
LLMScheduler: scheduler,
})
if err != nil {
t.Fatalf("Propose error: %v", err)
}
if scheduler.runs != 1 {
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
}
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "Hello, world" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "grammar", "*proposal*response-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "grammar"))
}
for _, f := range diagFiles {
raw, readErr := os.ReadFile(f)
if readErr != nil {
t.Fatalf("read diag %q: %v", f, readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
}
}
}

View File

@@ -0,0 +1,85 @@
package grammar
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages mirrors the Python grammar-module prompt intent:
// punctuation/capitalization/spacing cleanup only, with strict meaning guards.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: 0,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
system := "You are Audita, a conservative grammar cleanup assistant. Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. Do not change content, substitute words, or rewrite the speaker's phrasing."
user := "Review this transcript section and return only grammar cleanup corrections that should be applied.\n\n" +
"Rules:\n" +
"- Allowed changes are punctuation, capitalization, spacing, and article cleanup only.\n" +
"- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.\n" +
"- You may change the whole-word article \"a\" to \"an\" or \"an\" to \"a\" when the surrounding text otherwise stays the same.\n" +
"- Homophone, spoken-form, and mistranscription corrections are handled during a later review stage; do not propose them here.\n" +
"- Do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.\n" +
"- Do not change one written word into a different written word, except for capitalization changes to the same letters.\n" +
"- If a possible correction depends on changing a content word into a different word, omit it here rather than bundling it together with formatting cleanup.\n" +
"- Treat glossary names and aliases as protected spellings and context.\n" +
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n" +
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" +
"- If a segment includes categories, treat them as additional transcript context.\n" +
"- Use the exact id from the input segment.\n" +
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" +
"- Choose an original_text span that appears exactly once in the current segment text.\n" +
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" +
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" +
"- Do not return corrections where original_text and corrected_text are identical.\n" +
"- Do not return speaker, start, or end fields.\n" +
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
}, nil
}