eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 15 spoken-word module

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 11:56:28 +00:00
parent dbf3605712
commit a9f7fa27ff
10 changed files with 945 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
internal/modules/spoken_word/module.go Normal file
View File

@@ -0,0 +1,76 @@
package spoken_word
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type Module struct {
validators []contracts.Validator
}
func New() (*Module, error) {
spokenWordReview, err := validators.NewLLMBackedValidator("spoken_word_review", validators.LLMValidatorTypeSpokenWordReview, "")
if err != nil {
return nil, err
}
meaningReversal, err := validators.NewLLMBackedValidator("meaning_reversal_review", validators.LLMValidatorTypeMeaningReversal, "")
if err != nil {
return nil, err
}
return &Module{
validators: []contracts.Validator{
validators.NoEffectValidator{},
validators.OriginalTextPresenceValidator{},
validators.ConfidenceThresholdValidator{},
validators.ProtectedGlossaryTermValidator{},
validators.NonEmptyCorrectionValidator{},
spokenWordReview,
meaningReversal,
},
}, nil
}
func (m *Module) Key() string { return "spoken_word" }
func (m *Module) ReplacementPolicy() proposals.ReplacementPolicy {
// Python spoken_word module uses require_unique for conservative single-span replacement.
return proposals.ReplacementPolicyRequireUnique
}
func (m *Module) Validators() []contracts.Validator {
return append([]contracts.Validator(nil), m.validators...)
}
func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) {
messages, err := BuildProposalMessages(req.WorkingTranscript, req.Glossary)
if err != nil {
return nil, err
}
generated, err := proposal_generation.GenerateCandidates(ctx, proposal_generation.Request{
ModuleKey: req.RunSpec.ModuleKey,
ModuleInstance: req.RunSpec.InstanceName,
ReplacementPolicy: req.RunSpec.ReplacementPolicy,
WorkingTranscript: req.WorkingTranscript,
Section: req.Section,
Glossary: req.Glossary,
Config: req.Config,
Messages: messages,
StageName: fmt.Sprintf("%s:proposal", req.RunSpec.InstanceName),
StartIndex: 0,
LLMClient: req.LLMClient,
Scheduler: req.LLMScheduler,
DiagnosticsDir: req.DiagnosticsDir,
})
if err != nil {
return nil, err
}
return generated.Corrections, nil
}

295
internal/modules/spoken_word/module_test.go Normal file
View File

@@ -0,0 +1,295 @@
package spoken_word
import (
"context"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
)
type fakeLLMClient struct {
responses []proposal_generation.StructuredCorrectionSet
err error
calls []contracts.StructuredCompletionRequest
}
func (f *fakeLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
f.calls = append(f.calls, req)
if f.err != nil {
return contracts.StructuredCompletionResponse{}, f.err
}
target, ok := out.(*proposal_generation.StructuredCorrectionSet)
if !ok {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
}
if len(f.responses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
}
*target = f.responses[0]
f.responses = f.responses[1:]
return contracts.StructuredCompletionResponse{}, nil
}
type countingScheduler struct {
runs int
}
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
s.runs++
return fn(ctx)
}
func tinyTranscript() *schema.Transcript {
return &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "I I think, you know, we should go.", Categories: []string{"session"}},
}}
}
func tinyGlossary() *schema.Glossary {
return &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Aliases: []string{"Jester"}, Category: "faction", Summary: "Guild members", Plural: "Jesters"},
}}
}
func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary())
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
if len(msgs) != 2 {
t.Fatalf("expected 2 messages, got %d", len(msgs))
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, want := range []string{
"Protected glossary/context:",
"Transcript section:",
`"Aliases":`,
`"Category": "faction"`,
`"Summary": "Guild members"`,
`"Plural": "Jesters"`,
"conservative spoken-word cleanup assistant",
"Preserve substantive meaning",
"Do not paraphrase, summarize, reorder ideas",
`"id": 1`,
} {
if !strings.Contains(combined, want) {
t.Fatalf("expected prompt to contain %q", want)
}
}
for _, forbidden := range []string{
"style rewriting",
"invent",
"grammar-only cleanup",
"punctuation-only cleanup",
} {
if strings.Contains(strings.ToLower(combined), forbidden) {
t.Fatalf("prompt should not invite %q", forbidden)
}
}
}
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
if m.ReplacementPolicy() != proposals.ReplacementPolicyRequireUnique {
t.Fatalf("unexpected replacement policy: %q", m.ReplacementPolicy())
}
}
func TestSpokenWordModuleValidatorChain(t *testing.T) {
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
got := make([]string, 0, len(m.Validators()))
for _, v := range m.Validators() {
got = append(got, v.Name())
}
want := []string{
"no_effect",
"original_text_presence",
"confidence_threshold",
"protected_glossary_terms",
"non_empty_correction",
"spoken_word_review",
"meaning_reversal_review",
}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("unexpected validator chain\n got: %v\nwant: %v", got, want)
}
}
func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
secret := "spoken-word-secret"
client := &fakeLLMClient{
responses: []proposal_generation.StructuredCorrectionSet{
{
Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "I I think", CorrectedText: "I think", Confidence: 0.94},
},
},
},
}
scheduler := &countingScheduler{}
cfg := config.Default()
cfg.PrimaryLLM.APIKey = secret
m, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
diagDir := t.TempDir()
out, err := m.Propose(context.Background(), contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{
Config: &cfg,
WorkingTranscript: tinyTranscript(),
Glossary: tinyGlossary(),
DiagnosticsDir: diagDir,
},
RunSpec: contracts.ModuleRunSpec{
ModuleKey: "spoken_word",
InstanceName: "spoken_word",
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
},
LLMClient: client,
LLMScheduler: scheduler,
})
if err != nil {
t.Fatalf("Propose error: %v", err)
}
if scheduler.runs != 1 {
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
}
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
}
if len(out) != 1 || out[0].CorrectedText != "I think" {
t.Fatalf("unexpected proposals: %+v", out)
}
diagFiles, globErr := filepath.Glob(filepath.Join(diagDir, "spoken_word", "*proposal*response-payload.json"))
if globErr != nil {
t.Fatalf("glob error: %v", globErr)
}
if len(diagFiles) == 0 {
t.Fatalf("expected diagnostics response payload under %s", filepath.Join(diagDir, "spoken_word"))
}
for _, f := range diagFiles {
raw, readErr := os.ReadFile(f)
if readErr != nil {
t.Fatalf("read diag %q: %v", f, readErr)
}
if strings.Contains(string(raw), secret) {
t.Fatalf("secret leaked in diagnostics: %s", string(raw))
}
}
}
func TestSpokenWordConfidenceThresholdIsUsed(t *testing.T) {
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
var thresholdValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "confidence_threshold" {
thresholdValidator = v
break
}
}
if thresholdValidator == nil {
t.Fatal("expected confidence_threshold validator")
}
cfg := config.Default()
cfg.Thresholds.SpokenWord = 0.95
result, err := thresholdValidator.Validate(context.Background(), validators.Request{
Config: &cfg,
ModuleKey: "spoken_word",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1,
OriginalText: "I I think",
CorrectedText: "I think",
Confidence: 0.94,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
},
},
})
if err != nil {
t.Fatalf("validate error: %v", err)
}
if len(result.Decisions) != 1 {
t.Fatalf("expected one decision, got %+v", result.Decisions)
}
if result.Decisions[0].Approved {
t.Fatalf("expected rejection under spoken_word threshold, got %+v", result.Decisions[0])
}
if result.Decisions[0].ReasonCode != validators.ReasonLowConfidence {
t.Fatalf("expected low confidence reason, got %+v", result.Decisions[0])
}
}
func TestSpokenWordProtectedGlossaryTermBehaviorRejectsUnsafeCorrection(t *testing.T) {
module, err := New()
if err != nil {
t.Fatalf("New error: %v", err)
}
var protectedValidator contracts.Validator
for _, v := range module.Validators() {
if v.Name() == "protected_glossary_terms" {
protectedValidator = v
break
}
}
if protectedValidator == nil {
t.Fatal("expected protected_glossary_terms validator")
}
working := &schema.Transcript{Segments: []schema.Segment{
{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "Jesters are here."},
}}
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{
{Name: "Jesters", Category: "faction", Summary: "Faction"},
}}
result, err := protectedValidator.Validate(context.Background(), validators.Request{
WorkingTranscript: working,
Glossary: glossary,
ModuleKey: "spoken_word",
CandidateProposal: []proposals.EnrichedCorrectionProposal{
{
CorrectionProposal: proposals.CorrectionProposal{
TargetSegmentID: 1,
OriginalText: "Jesters",
CorrectedText: "gestures",
Confidence: 0.99,
},
ProposalMetadata: proposals.ProposalMetadata{ProposalIndex: 0, ModuleKey: "spoken_word", ModuleInstance: "spoken_word"},
},
},
})
if err != nil {
t.Fatalf("validate error: %v", err)
}
if len(result.Decisions) != 1 {
t.Fatalf("expected one decision, got %+v", result.Decisions)
}
if result.Decisions[0].Approved {
t.Fatalf("expected protected-term rejection, got %+v", result.Decisions[0])
}
if result.Decisions[0].ReasonCode != validators.ReasonProtectedGlossaryTerm {
t.Fatalf("expected protected glossary-term reason, got %+v", result.Decisions[0])
}
}

86
internal/modules/spoken_word/prompt.go Normal file
View File

@@ -0,0 +1,86 @@
package spoken_word
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
)
type promptSegment struct {
ID int `json:"id"`
Speaker string `json:"speaker"`
Start float64 `json:"start"`
End float64 `json:"end"`
Text string `json:"text"`
Categories []string `json:"categories,omitempty"`
}
type promptTranscriptSection struct {
SectionIndex int `json:"section_index"`
Segments []promptSegment `json:"segments"`
}
// BuildProposalMessages mirrors the Python spoken_word-module prompt intent:
// conservative dysfluency cleanup with strict semantic preservation.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
}
sectionPayload := promptTranscriptSection{
SectionIndex: 0,
Segments: make([]promptSegment, 0),
}
if transcript != nil {
for _, s := range transcript.Segments {
sectionPayload.Segments = append(sectionPayload.Segments, promptSegment{
ID: s.ID,
Speaker: s.Speaker,
Start: s.Start,
End: s.End,
Text: s.Text,
Categories: append([]string(nil), s.Categories...),
})
}
}
sectionJSON, err := json.MarshalIndent(sectionPayload, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
}
system := "You are Audita, a conservative spoken-word cleanup assistant. Identify only low-risk cleanup of repeated words or short phrases, filler words, hesitation artifacts, and similar dysfluencies that commonly appear in spoken English transcripts. Preserve substantive meaning, named entities, and transcript content."
user := "Review this transcript section and return only spoken-word cleanup corrections that should be applied.\n\n" +
"Rules:\n" +
"- Approve only conservative cleanup of repeated words, repeated short phrases, filler words, hesitation artifacts, and similar spoken dysfluencies.\n" +
"- You may collapse adjacent repetition such as \"I I think\" to \"I think\" or remove filler spans such as \"you know\" or \"uh\" when local context supports that cleanup.\n" +
"- Do not collapse repeated words or short phrases when the repetition plausibly expresses urgency, excitement, insistence, or deliberate rhetorical emphasis rather than dysfluency.\n" +
"- Phrases such as \"Help! Help! Help!\", \"Stop! Stop! Stop!\", \"No! No! No!\", \"Yes! Yes! Yes!\", and \"Go! Go! Go!\" are often intentional emphasis and should usually be preserved.\n" +
"- Only collapse repetition when local context supports it as accidental spoken repetition, hesitation, or verbal restart.\n" +
"- You may include low-risk punctuation, spacing, or capitalization cleanup when it is part of removing a dysfluency, such as removing ellipses or hesitation punctuation that no longer belongs after the cleanup.\n" +
"- Do not paraphrase, summarize, reorder ideas, replace content with different wording, or make substantive semantic edits.\n" +
"- Do not change clear content words just because a different phrasing reads better.\n" +
"- Do not convert uncertain statements into certain statements.\n" +
"- Treat glossary names and aliases as protected spellings and context.\n" +
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases that already appear correctly in the transcript.\n" +
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n" +
"- If a segment includes categories, treat them as additional transcript context.\n" +
"- Use the exact id from the input segment.\n" +
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" +
"- Choose an original_text span that appears exactly once in the current segment text.\n" +
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" +
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n" +
"- Do not return corrections where original_text and corrected_text are identical.\n" +
"- Do not return speaker, start, or end fields.\n" +
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{
{Role: "system", Content: system},
{Role: "user", Content: user},
}, nil
}