Implemented the grammar LLM review module
This commit is contained in:
13
README.md
13
README.md
@@ -4,7 +4,7 @@ Audita is a framework-first transcript correction application. The public `audit
|
|||||||
|
|
||||||
- deterministic transcript normalization
|
- deterministic transcript normalization
|
||||||
- token-batched module orchestration
|
- token-batched module orchestration
|
||||||
- concrete `glossary`, `homophones`, and `spoken_word` modules built on reusable proposal / validator contracts
|
- concrete `glossary`, `homophones`, `spoken_word`, and `grammar` modules built on reusable proposal / validator contracts
|
||||||
- structured run reporting and work-dir diagnostics
|
- structured run reporting and work-dir diagnostics
|
||||||
|
|
||||||
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
The previous working implementation has been preserved as `audita_prototype` inside this repository. Its full regression suite lives under `tests/audita_prototype`.
|
||||||
@@ -42,10 +42,12 @@ Resolved run instance names are auto-numbered for repeats, so the default report
|
|||||||
4. `spoken_word`
|
4. `spoken_word`
|
||||||
5. `grammar`
|
5. `grammar`
|
||||||
|
|
||||||
The default module sequence is partially implemented today:
|
The default module sequence is fully implemented today:
|
||||||
|
|
||||||
- `glossary`, `homophones`, the second `glossary` pass, and `spoken_word` run real LLM-backed proposal and validation stages
|
- `glossary` proposes glossary-supported acoustic corrections
|
||||||
- `grammar` remains a stub and currently proposes no corrections
|
- `homophones` proposes conservative homophone and mistranscription corrections
|
||||||
|
- `spoken_word` proposes conservative dysfluency cleanup
|
||||||
|
- `grammar` proposes punctuation, capitalization, and spacing cleanup only
|
||||||
|
|
||||||
To run a custom module sequence, pass `--modules`:
|
To run a custom module sequence, pass `--modules`:
|
||||||
|
|
||||||
@@ -77,7 +79,7 @@ audita process transcript.json --glossary glossary.yaml --output corrected.json
|
|||||||
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
Without `--output`, Audita writes the corrected transcript JSON to stdout and progress logs to stderr.
|
||||||
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
`--report-json` writes a separate machine-readable run report and never mixes report data into stdout.
|
||||||
|
|
||||||
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, and `spoken_word` modules make real LLM calls.
|
Useful configuration can be supplied by CLI flag or environment variable. CLI flags take precedence over environment variables. Normal runs now require `OPENROUTER_API_KEY`, because the `glossary`, `homophones`, `spoken_word`, and `grammar` modules make real LLM calls.
|
||||||
|
|
||||||
| Environment variable | CLI flag | Default | Purpose |
|
| Environment variable | CLI flag | Default | Purpose |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
@@ -87,6 +89,7 @@ Useful configuration can be supplied by CLI flag or environment variable. CLI fl
|
|||||||
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
| `AUDITA_MAX_RETRIES` | `--max-retries` | `3` | Maximum Instructor retries for structured responses |
|
||||||
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch |
|
| `AUDITA_MAX_SECTION_TOKENS` | `--max-section-tokens` | `6144` | Maximum estimated tokens per transcript batch |
|
||||||
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
| `AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD` | `--glossary-confidence-threshold` | `0.8` | Minimum confidence required for glossary proposals to survive validation |
|
||||||
|
| `AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD` | `--grammar-confidence-threshold` | `0.8` | Minimum confidence required for grammar proposals to survive validation |
|
||||||
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
| `AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD` | `--homophones-confidence-threshold` | `0.8` | Minimum confidence required for homophone proposals to survive validation |
|
||||||
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
| `AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD` | `--spoken-word-confidence-threshold` | `0.8` | Minimum confidence required for spoken-word proposals to survive validation |
|
||||||
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
| `AUDITA_NORMALIZE_MAX_SEGMENT_GAP` | `--normalize-max-segment-gap` | `4.0` | Same-speaker gaps eligible for deterministic merging |
|
||||||
|
|||||||
@@ -40,6 +40,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||||||
type=float,
|
type=float,
|
||||||
help="minimum confidence required for glossary proposals to survive validation",
|
help="minimum confidence required for glossary proposals to survive validation",
|
||||||
)
|
)
|
||||||
|
process.add_argument(
|
||||||
|
"--grammar-confidence-threshold",
|
||||||
|
type=float,
|
||||||
|
help="minimum confidence required for grammar proposals to survive validation",
|
||||||
|
)
|
||||||
process.add_argument(
|
process.add_argument(
|
||||||
"--homophones-confidence-threshold",
|
"--homophones-confidence-threshold",
|
||||||
type=float,
|
type=float,
|
||||||
@@ -89,6 +94,7 @@ def _process(args: argparse.Namespace) -> int:
|
|||||||
max_retries=args.max_retries,
|
max_retries=args.max_retries,
|
||||||
max_section_tokens=args.max_section_tokens,
|
max_section_tokens=args.max_section_tokens,
|
||||||
glossary_confidence_threshold=args.glossary_confidence_threshold,
|
glossary_confidence_threshold=args.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=args.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=args.homophones_confidence_threshold,
|
homophones_confidence_threshold=args.homophones_confidence_threshold,
|
||||||
spoken_word_confidence_threshold=args.spoken_word_confidence_threshold,
|
spoken_word_confidence_threshold=args.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=args.normalize_max_segment_gap,
|
normalize_max_segment_gap=args.normalize_max_segment_gap,
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
|
|||||||
DEFAULT_MAX_RETRIES = 3
|
DEFAULT_MAX_RETRIES = 3
|
||||||
DEFAULT_MAX_SECTION_TOKENS = 6144
|
DEFAULT_MAX_SECTION_TOKENS = 6144
|
||||||
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
|
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD = 0.80
|
||||||
|
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD = 0.80
|
||||||
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80
|
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD = 0.80
|
||||||
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80
|
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD = 0.80
|
||||||
DEFAULT_WORK_DIR = "/tmp/audita"
|
DEFAULT_WORK_DIR = "/tmp/audita"
|
||||||
@@ -32,6 +33,7 @@ class ConfigOverrides:
|
|||||||
max_retries: Optional[int] = None
|
max_retries: Optional[int] = None
|
||||||
max_section_tokens: Optional[int] = None
|
max_section_tokens: Optional[int] = None
|
||||||
glossary_confidence_threshold: Optional[float] = None
|
glossary_confidence_threshold: Optional[float] = None
|
||||||
|
grammar_confidence_threshold: Optional[float] = None
|
||||||
homophones_confidence_threshold: Optional[float] = None
|
homophones_confidence_threshold: Optional[float] = None
|
||||||
spoken_word_confidence_threshold: Optional[float] = None
|
spoken_word_confidence_threshold: Optional[float] = None
|
||||||
normalize_max_segment_gap: Optional[float] = None
|
normalize_max_segment_gap: Optional[float] = None
|
||||||
@@ -51,6 +53,7 @@ class AuditaConfig:
|
|||||||
max_retries: int = DEFAULT_MAX_RETRIES
|
max_retries: int = DEFAULT_MAX_RETRIES
|
||||||
max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS
|
max_section_tokens: int = DEFAULT_MAX_SECTION_TOKENS
|
||||||
glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
|
glossary_confidence_threshold: float = DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
|
||||||
|
grammar_confidence_threshold: float = DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
|
||||||
homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
|
homophones_confidence_threshold: float = DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
|
||||||
spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
|
spoken_word_confidence_threshold: float = DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
|
||||||
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
||||||
@@ -96,6 +99,12 @@ class AuditaConfig:
|
|||||||
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
|
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
|
||||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
|
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
|
||||||
),
|
),
|
||||||
|
grammar_confidence_threshold=_select_float(
|
||||||
|
selected.grammar_confidence_threshold,
|
||||||
|
source.get("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD"),
|
||||||
|
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
|
||||||
|
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD",
|
||||||
|
),
|
||||||
homophones_confidence_threshold=_select_float(
|
homophones_confidence_threshold=_select_float(
|
||||||
selected.homophones_confidence_threshold,
|
selected.homophones_confidence_threshold,
|
||||||
source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"),
|
source.get("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD"),
|
||||||
@@ -156,6 +165,8 @@ class AuditaConfig:
|
|||||||
raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.")
|
raise AuditaConfigError("AUDITA_MAX_SECTION_TOKENS must be greater than zero.")
|
||||||
if not 0.0 <= self.glossary_confidence_threshold <= 1.0:
|
if not 0.0 <= self.glossary_confidence_threshold <= 1.0:
|
||||||
raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
|
raise AuditaConfigError("AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
|
||||||
|
if not 0.0 <= self.grammar_confidence_threshold <= 1.0:
|
||||||
|
raise AuditaConfigError("AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
|
||||||
if not 0.0 <= self.homophones_confidence_threshold <= 1.0:
|
if not 0.0 <= self.homophones_confidence_threshold <= 1.0:
|
||||||
raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
|
raise AuditaConfigError("AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0.")
|
||||||
if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0:
|
if not 0.0 <= self.spoken_word_confidence_threshold <= 1.0:
|
||||||
@@ -190,6 +201,7 @@ class AuditaConfig:
|
|||||||
"max_retries": self.max_retries,
|
"max_retries": self.max_retries,
|
||||||
"max_section_tokens": self.max_section_tokens,
|
"max_section_tokens": self.max_section_tokens,
|
||||||
"glossary_confidence_threshold": self.glossary_confidence_threshold,
|
"glossary_confidence_threshold": self.glossary_confidence_threshold,
|
||||||
|
"grammar_confidence_threshold": self.grammar_confidence_threshold,
|
||||||
"homophones_confidence_threshold": self.homophones_confidence_threshold,
|
"homophones_confidence_threshold": self.homophones_confidence_threshold,
|
||||||
"spoken_word_confidence_threshold": self.spoken_word_confidence_threshold,
|
"spoken_word_confidence_threshold": self.spoken_word_confidence_threshold,
|
||||||
"normalize_max_segment_gap": self.normalize_max_segment_gap,
|
"normalize_max_segment_gap": self.normalize_max_segment_gap,
|
||||||
|
|||||||
@@ -2,7 +2,15 @@ from typing import Sequence
|
|||||||
|
|
||||||
from audita.core.chunking import TranscriptSection
|
from audita.core.chunking import TranscriptSection
|
||||||
from audita.framework.models import CorrectionProposal, ModuleContext
|
from audita.framework.models import CorrectionProposal, ModuleContext
|
||||||
from audita.validators import ProtectedGlossaryTermsValidator, Validator
|
from audita.framework.proposal_generation import generate_llm_correction_proposals
|
||||||
|
from audita.modules.prompts import build_grammar_proposal_messages
|
||||||
|
from audita.validators import (
|
||||||
|
GrammarOnlyValidator,
|
||||||
|
MeaningReversalValidator,
|
||||||
|
ProposalConfidenceValidator,
|
||||||
|
ProtectedGlossaryTermsValidator,
|
||||||
|
Validator,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class GrammarModule:
|
class GrammarModule:
|
||||||
@@ -10,11 +18,20 @@ class GrammarModule:
|
|||||||
replacement_policy = "require_unique"
|
replacement_policy = "require_unique"
|
||||||
|
|
||||||
def validators(self) -> Sequence[Validator]:
|
def validators(self) -> Sequence[Validator]:
|
||||||
return [ProtectedGlossaryTermsValidator("protected_glossary_guard")]
|
return [
|
||||||
|
ProposalConfidenceValidator("proposal_confidence_guard", "grammar_confidence_threshold"),
|
||||||
|
ProtectedGlossaryTermsValidator("protected_glossary_guard"),
|
||||||
|
GrammarOnlyValidator("grammar_only_guard"),
|
||||||
|
MeaningReversalValidator("meaning_reversal_review"),
|
||||||
|
]
|
||||||
|
|
||||||
def propose(
|
def propose(
|
||||||
self,
|
self,
|
||||||
transcript_section: TranscriptSection,
|
transcript_section: TranscriptSection,
|
||||||
context: ModuleContext,
|
context: ModuleContext,
|
||||||
) -> Sequence[CorrectionProposal]:
|
) -> Sequence[CorrectionProposal]:
|
||||||
return []
|
return generate_llm_correction_proposals(
|
||||||
|
section=transcript_section,
|
||||||
|
context=context,
|
||||||
|
prompt_builder=build_grammar_proposal_messages,
|
||||||
|
)
|
||||||
|
|||||||
@@ -122,3 +122,38 @@ def build_spoken_word_proposal_messages(section: TranscriptSection, glossary: Gl
|
|||||||
f"Transcript section:\n{section_json}"
|
f"Transcript section:\n{section_json}"
|
||||||
)
|
)
|
||||||
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
||||||
|
|
||||||
|
|
||||||
|
def build_grammar_proposal_messages(section: TranscriptSection, glossary: Glossary) -> List[Message]:
|
||||||
|
glossary_json = json.dumps(glossary.model_dump(mode="json", exclude_none=True), ensure_ascii=False, indent=2)
|
||||||
|
section_json = json.dumps(section.prompt_payload(), ensure_ascii=False, indent=2)
|
||||||
|
|
||||||
|
system = (
|
||||||
|
"You are Audita, a conservative grammar cleanup assistant. "
|
||||||
|
"Identify only punctuation, capitalization, and spacing cleanup that preserves the same underlying words. "
|
||||||
|
"Do not change content, substitute words, or rewrite the speaker's phrasing."
|
||||||
|
)
|
||||||
|
user = (
|
||||||
|
"Review this transcript section and return only grammar cleanup corrections that should be applied.\n\n"
|
||||||
|
"Rules:\n"
|
||||||
|
"- Allowed changes are punctuation, capitalization, and spacing cleanup only.\n"
|
||||||
|
"- You may add, remove, or adjust commas, periods, quotation marks, apostrophes, dashes, ellipses, spacing, and capitalization when the underlying words stay the same.\n"
|
||||||
|
"- Do not make word substitutions, spelling fixes, homophone fixes, filler cleanup, repetition cleanup, paraphrases, or other semantic rewrites.\n"
|
||||||
|
"- Do not change one written word into a different written word, except for capitalization changes to the same letters.\n"
|
||||||
|
"- Treat glossary names and aliases as protected spellings and context.\n"
|
||||||
|
"- Do not replace, Anglicize, normalize, lowercase, or otherwise alter protected glossary names or aliases away from their glossary spelling.\n"
|
||||||
|
"- Preserve canonical glossary capitalization for protected names and aliases, even if they look unusual.\n"
|
||||||
|
"- Use the exact id from the input segment.\n"
|
||||||
|
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
|
||||||
|
"- Choose an original_text span that appears exactly once in the current segment text.\n"
|
||||||
|
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
|
||||||
|
"- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
|
||||||
|
"- Do not return corrections where original_text and corrected_text are identical.\n"
|
||||||
|
"- Do not return speaker, start, or end fields.\n"
|
||||||
|
"- Return only changed segments; do not return entries for unchanged segments.\n"
|
||||||
|
"- confidence must be between 0.0 and 1.0.\n"
|
||||||
|
"- If no corrections are needed, return an empty corrections list.\n\n"
|
||||||
|
f"Protected glossary/context:\n{glossary_json}\n\n"
|
||||||
|
f"Transcript section:\n{section_json}"
|
||||||
|
)
|
||||||
|
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
|
from .base import ValidationContext, ValidationDecision, ValidationResult, Validator
|
||||||
from .deterministic import ProposalConfidenceValidator, ProtectedGlossaryTermsValidator
|
from .deterministic import GrammarOnlyValidator, ProposalConfidenceValidator, ProtectedGlossaryTermsValidator
|
||||||
from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
|
from .llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
|
||||||
from .protection import ProtectedVocabulary
|
from .protection import ProtectedVocabulary
|
||||||
|
|
||||||
@@ -10,6 +10,7 @@ __all__ = [
|
|||||||
"Validator",
|
"Validator",
|
||||||
"ProposalConfidenceValidator",
|
"ProposalConfidenceValidator",
|
||||||
"ProtectedGlossaryTermsValidator",
|
"ProtectedGlossaryTermsValidator",
|
||||||
|
"GrammarOnlyValidator",
|
||||||
"ProtectedVocabulary",
|
"ProtectedVocabulary",
|
||||||
"SpokenFormPlausibilityValidator",
|
"SpokenFormPlausibilityValidator",
|
||||||
"SpokenWordValidator",
|
"SpokenWordValidator",
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import string
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
from .base import ValidationContext, ValidationDecision, ValidationResult
|
from .base import ValidationContext, ValidationDecision, ValidationResult
|
||||||
@@ -48,3 +49,38 @@ class ProtectedGlossaryTermsValidator:
|
|||||||
for proposal in context.proposals
|
for proposal in context.proposals
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_GRAMMAR_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GrammarOnlyValidator:
|
||||||
|
name: str
|
||||||
|
execution_kind: str = "deterministic"
|
||||||
|
|
||||||
|
def validate(self, context: ValidationContext) -> ValidationResult:
|
||||||
|
return ValidationResult(
|
||||||
|
validator_name=self.name,
|
||||||
|
execution_kind=self.execution_kind,
|
||||||
|
decisions=[
|
||||||
|
ValidationDecision(
|
||||||
|
proposal_index=proposal.proposal_index,
|
||||||
|
approved=_grammar_semantic_key(proposal.original_text) == _grammar_semantic_key(proposal.corrected_text),
|
||||||
|
reason=(
|
||||||
|
None
|
||||||
|
if _grammar_semantic_key(proposal.original_text) == _grammar_semantic_key(proposal.corrected_text)
|
||||||
|
else "correction is not limited to punctuation, capitalization, and spacing"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for proposal in context.proposals
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _grammar_semantic_key(text: str) -> str:
|
||||||
|
return "".join(
|
||||||
|
character.casefold()
|
||||||
|
for character in text
|
||||||
|
if not character.isspace() and character not in _GRAMMAR_PUNCTUATION
|
||||||
|
)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from audita.core.config import AuditaConfig
|
|||||||
from audita.core.errors import AuditaLLMError
|
from audita.core.errors import AuditaLLMError
|
||||||
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
from audita.core.schemas import parse_glossary_yaml, parse_transcript_json
|
||||||
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
from audita.framework.models import CorrectionProposal, ModuleRunSpec
|
||||||
|
from audita.validators import GrammarOnlyValidator
|
||||||
from audita.validators.base import ValidationContext
|
from audita.validators.base import ValidationContext
|
||||||
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
|
from audita.validators.llm import MeaningReversalValidator, SpokenFormPlausibilityValidator, SpokenWordValidator
|
||||||
from audita.validators.prompts import (
|
from audita.validators.prompts import (
|
||||||
@@ -309,6 +310,118 @@ def test_spoken_word_validator_allows_punctuation_cleanup_tied_to_dysfluency(tmp
|
|||||||
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
|
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [(0, True)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_only_validator_allows_formatting_only_changes(tmp_path):
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "hello there"},
|
||||||
|
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "cant we go"},
|
||||||
|
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "Hello,world"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
proposals = [
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=0,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=1,
|
||||||
|
original_text="hello there",
|
||||||
|
corrected_text="Hello there.",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=1,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=2,
|
||||||
|
original_text="cant",
|
||||||
|
corrected_text="can't",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=2,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=3,
|
||||||
|
original_text="Hello,world",
|
||||||
|
corrected_text="Hello, world",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
result = GrammarOnlyValidator("grammar_only_guard").validate(
|
||||||
|
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [(decision.proposal_index, decision.approved) for decision in result.decisions] == [
|
||||||
|
(0, True),
|
||||||
|
(1, True),
|
||||||
|
(2, True),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_only_validator_rejects_word_level_changes(tmp_path):
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "their plan"},
|
||||||
|
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "dam"},
|
||||||
|
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "uh"},
|
||||||
|
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "I I agree"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
proposals = [
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=0,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=1,
|
||||||
|
original_text="their",
|
||||||
|
corrected_text="there",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=1,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=2,
|
||||||
|
original_text="dam",
|
||||||
|
corrected_text="damn",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=2,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=3,
|
||||||
|
original_text="uh",
|
||||||
|
corrected_text="",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
CorrectionProposal(
|
||||||
|
proposal_index=3,
|
||||||
|
module_instance="grammar",
|
||||||
|
module_key="grammar",
|
||||||
|
id=4,
|
||||||
|
original_text="I I",
|
||||||
|
corrected_text="I",
|
||||||
|
confidence=0.95,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
result = GrammarOnlyValidator("grammar_only_guard").validate(
|
||||||
|
_context(proposals=proposals, transcript=transcript, llm_client=None, tmp_path=tmp_path)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [decision.approved for decision in result.decisions] == [False, False, False, False]
|
||||||
|
assert all(
|
||||||
|
decision.reason == "correction is not limited to punctuation, capitalization, and spacing"
|
||||||
|
for decision in result.decisions
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("validator", "payload", "message_fragment"),
|
("validator", "payload", "message_fragment"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -5,9 +5,14 @@ from audita.core.config import AuditaConfig
|
|||||||
from audita.core.errors import AuditaLLMError
|
from audita.core.errors import AuditaLLMError
|
||||||
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
|
from audita.core.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
|
||||||
from audita.framework.models import ModuleContext, ModuleRunSpec
|
from audita.framework.models import ModuleContext, ModuleRunSpec
|
||||||
|
from audita.modules.grammar import GrammarModule
|
||||||
from audita.modules.glossary import GlossaryModule
|
from audita.modules.glossary import GlossaryModule
|
||||||
from audita.modules.homophones import HomophonesModule
|
from audita.modules.homophones import HomophonesModule
|
||||||
from audita.modules.prompts import build_homophones_proposal_messages, build_spoken_word_proposal_messages
|
from audita.modules.prompts import (
|
||||||
|
build_grammar_proposal_messages,
|
||||||
|
build_homophones_proposal_messages,
|
||||||
|
build_spoken_word_proposal_messages,
|
||||||
|
)
|
||||||
from audita.modules.spoken_word import SpokenWordModule
|
from audita.modules.spoken_word import SpokenWordModule
|
||||||
from audita.pipeline import process_transcript_result
|
from audita.pipeline import process_transcript_result
|
||||||
|
|
||||||
@@ -175,6 +180,70 @@ def test_spoken_word_prompt_is_explicitly_scoped_to_dysfluency_cleanup():
|
|||||||
assert '"id": 1' in messages[1]["content"]
|
assert '"id": 1' in messages[1]["content"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_module_propose_writes_diagnostics_and_returns_proposals_without_api_key(tmp_path):
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
||||||
|
module = GrammarModule()
|
||||||
|
client = FakeStructuredLLMClient(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"corrections": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"original_text": "hello world",
|
||||||
|
"corrected_text": "Hello world.",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
context = ModuleContext(
|
||||||
|
run_spec=ModuleRunSpec(instance_name="grammar", module_key="grammar", module=module),
|
||||||
|
glossary=_glossary(),
|
||||||
|
config=AuditaConfig.from_sources(env={}),
|
||||||
|
run_dir=tmp_path,
|
||||||
|
llm_client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
proposals = list(module.propose(section, context))
|
||||||
|
|
||||||
|
assert [(proposal.id, proposal.original_text, proposal.corrected_text, proposal.confidence) for proposal in proposals] == [
|
||||||
|
(1, "hello world", "Hello world.", 0.95)
|
||||||
|
]
|
||||||
|
assert (tmp_path / "prompt-0000.json").exists()
|
||||||
|
assert (tmp_path / "corrections-0000.json").exists()
|
||||||
|
prompt_text = client.calls[0]["messages"][1]["content"]
|
||||||
|
assert "punctuation, capitalization, and spacing" in prompt_text
|
||||||
|
assert "exact text span" in prompt_text
|
||||||
|
assert "word substitutions" in prompt_text
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_prompt_is_explicitly_scoped_to_formatting_cleanup():
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
section = chunk_transcript(transcript, max_section_tokens=1000)[0]
|
||||||
|
|
||||||
|
messages = build_grammar_proposal_messages(section, _glossary())
|
||||||
|
combined = messages[0]["content"] + messages[1]["content"]
|
||||||
|
|
||||||
|
assert "punctuation, capitalization, and spacing" in combined
|
||||||
|
assert "word substitutions" in combined
|
||||||
|
assert "homophone fixes" in combined
|
||||||
|
assert '"id": 1' in messages[1]["content"]
|
||||||
|
|
||||||
|
|
||||||
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
|
def test_process_transcript_result_uses_injected_fake_client_and_applies_sequential_module_updates(tmp_path):
|
||||||
transcript = parse_source_transcript_json(
|
transcript = parse_source_transcript_json(
|
||||||
"""
|
"""
|
||||||
@@ -194,7 +263,9 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
|
|||||||
max_retries=config.max_retries,
|
max_retries=config.max_retries,
|
||||||
max_section_tokens=config.max_section_tokens,
|
max_section_tokens=config.max_section_tokens,
|
||||||
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
||||||
@@ -266,6 +337,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
|
|||||||
},
|
},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
|
{"corrections": []},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -281,6 +353,7 @@ def test_process_transcript_result_uses_injected_fake_client_and_applies_sequent
|
|||||||
"homophones:meaning_reversal_review",
|
"homophones:meaning_reversal_review",
|
||||||
"glossary_2:proposal",
|
"glossary_2:proposal",
|
||||||
"spoken_word:proposal",
|
"spoken_word:proposal",
|
||||||
|
"grammar:proposal",
|
||||||
]
|
]
|
||||||
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
|
assert "There were Jesters at the dam." in client.calls[3]["messages"][1]["content"]
|
||||||
|
|
||||||
@@ -301,7 +374,9 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
|
|||||||
max_retries=base_config.max_retries,
|
max_retries=base_config.max_retries,
|
||||||
max_section_tokens=base_config.max_section_tokens,
|
max_section_tokens=base_config.max_section_tokens,
|
||||||
glossary_confidence_threshold=0.96,
|
glossary_confidence_threshold=0.96,
|
||||||
|
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
||||||
@@ -324,6 +399,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
|
|||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
|
{"corrections": []},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -335,6 +411,7 @@ def test_process_transcript_result_rejects_below_threshold_proposals_before_llm_
|
|||||||
"homophones:proposal",
|
"homophones:proposal",
|
||||||
"glossary_2:proposal",
|
"glossary_2:proposal",
|
||||||
"spoken_word:proposal",
|
"spoken_word:proposal",
|
||||||
|
"grammar:proposal",
|
||||||
]
|
]
|
||||||
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
||||||
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
|
assert result.report.skipped_corrections[0].source == "validator:proposal_confidence_guard"
|
||||||
@@ -358,6 +435,7 @@ def test_process_transcript_result_runs_spoken_word_module_with_full_validator_c
|
|||||||
max_retries=base_config.max_retries,
|
max_retries=base_config.max_retries,
|
||||||
max_section_tokens=base_config.max_section_tokens,
|
max_section_tokens=base_config.max_section_tokens,
|
||||||
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
||||||
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
||||||
@@ -440,6 +518,7 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll
|
|||||||
max_retries=base_config.max_retries,
|
max_retries=base_config.max_retries,
|
||||||
max_section_tokens=base_config.max_section_tokens,
|
max_section_tokens=base_config.max_section_tokens,
|
||||||
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
||||||
spoken_word_confidence_threshold=0.96,
|
spoken_word_confidence_threshold=0.96,
|
||||||
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
||||||
@@ -476,3 +555,130 @@ def test_process_transcript_result_rejects_spoken_word_below_threshold_before_ll
|
|||||||
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
|
assert [call["stage_name"] for call in client.calls] == ["spoken_word:proposal"]
|
||||||
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
||||||
assert result.report.modules[0].validators[1].candidate_count == 0
|
assert result.report.modules[0].validators[1].candidate_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_transcript_result_runs_grammar_module_with_full_validator_chain(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
base_config = AuditaConfig.from_sources(env={})
|
||||||
|
config = AuditaConfig(
|
||||||
|
api_key=base_config.api_key,
|
||||||
|
model=base_config.model,
|
||||||
|
base_url=base_config.base_url,
|
||||||
|
max_retries=base_config.max_retries,
|
||||||
|
max_section_tokens=base_config.max_section_tokens,
|
||||||
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=base_config.grammar_confidence_threshold,
|
||||||
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
||||||
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
||||||
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
||||||
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
||||||
|
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
|
||||||
|
work_dir=tmp_path / "work",
|
||||||
|
work_dir_retention="always",
|
||||||
|
)
|
||||||
|
client = FakeStructuredLLMClient(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"corrections": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"original_text": "hello world",
|
||||||
|
"corrected_text": "Hello world.",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"validations": [
|
||||||
|
{
|
||||||
|
"correction_index": 0,
|
||||||
|
"approved": True,
|
||||||
|
"confidence": 0.99,
|
||||||
|
"reason": "Does not reverse the segment meaning.",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = process_transcript_result(
|
||||||
|
transcript,
|
||||||
|
_glossary(),
|
||||||
|
config,
|
||||||
|
module_keys=["grammar"],
|
||||||
|
llm_client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.transcript[0].text == "Hello world."
|
||||||
|
assert [call["stage_name"] for call in client.calls] == [
|
||||||
|
"grammar:proposal",
|
||||||
|
"grammar:meaning_reversal_review",
|
||||||
|
]
|
||||||
|
assert [validator["name"] for validator in result.report.modules[0].to_dict()["validators"]] == [
|
||||||
|
"proposal_confidence_guard",
|
||||||
|
"protected_glossary_guard",
|
||||||
|
"grammar_only_guard",
|
||||||
|
"meaning_reversal_review",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_transcript_result_rejects_grammar_below_threshold_before_later_validators(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "hello world"}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
base_config = AuditaConfig.from_sources(env={})
|
||||||
|
config = AuditaConfig(
|
||||||
|
api_key=base_config.api_key,
|
||||||
|
model=base_config.model,
|
||||||
|
base_url=base_config.base_url,
|
||||||
|
max_retries=base_config.max_retries,
|
||||||
|
max_section_tokens=base_config.max_section_tokens,
|
||||||
|
glossary_confidence_threshold=base_config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=0.96,
|
||||||
|
homophones_confidence_threshold=base_config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=base_config.spoken_word_confidence_threshold,
|
||||||
|
normalize_max_segment_gap=base_config.normalize_max_segment_gap,
|
||||||
|
normalize_ellipsis_gap=base_config.normalize_ellipsis_gap,
|
||||||
|
normalize_max_segment_duration=base_config.normalize_max_segment_duration,
|
||||||
|
normalize_max_segment_tokens=base_config.normalize_max_segment_tokens,
|
||||||
|
work_dir=tmp_path / "work",
|
||||||
|
work_dir_retention="always",
|
||||||
|
)
|
||||||
|
client = FakeStructuredLLMClient(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"corrections": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"original_text": "hello world",
|
||||||
|
"corrected_text": "Hello world.",
|
||||||
|
"confidence": 0.95,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result = process_transcript_result(
|
||||||
|
transcript,
|
||||||
|
_glossary(),
|
||||||
|
config,
|
||||||
|
module_keys=["grammar"],
|
||||||
|
llm_client=client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.transcript[0].text == "hello world"
|
||||||
|
assert [call["stage_name"] for call in client.calls] == ["grammar:proposal"]
|
||||||
|
assert result.report.skipped_corrections[0].reason == "proposal confidence below threshold"
|
||||||
|
assert result.report.modules[0].validators[1].candidate_count == 0
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ def test_process_help_exposes_framework_flags(capsys):
|
|||||||
assert "--max-retries" in output
|
assert "--max-retries" in output
|
||||||
assert "--max-section-tokens" in output
|
assert "--max-section-tokens" in output
|
||||||
assert "--glossary-confidence-threshold" in output
|
assert "--glossary-confidence-threshold" in output
|
||||||
|
assert "--grammar-confidence-threshold" in output
|
||||||
assert "--homophones-confidence-threshold" in output
|
assert "--homophones-confidence-threshold" in output
|
||||||
assert "--spoken-word-confidence-threshold" in output
|
assert "--spoken-word-confidence-threshold" in output
|
||||||
assert "--work-dir-retention" in output
|
assert "--work-dir-retention" in output
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from audita.core.config import (
|
|||||||
AuditaConfig,
|
AuditaConfig,
|
||||||
ConfigOverrides,
|
ConfigOverrides,
|
||||||
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
|
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
|
||||||
|
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
|
||||||
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
|
DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD,
|
||||||
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
|
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
|
||||||
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
|
DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD,
|
||||||
@@ -19,6 +20,7 @@ def test_default_config_allows_missing_api_key():
|
|||||||
assert config.api_key is None
|
assert config.api_key is None
|
||||||
assert config.module_keys == DEFAULT_MODULE_KEYS
|
assert config.module_keys == DEFAULT_MODULE_KEYS
|
||||||
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
|
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
|
||||||
|
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
|
||||||
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
|
assert config.homophones_confidence_threshold == DEFAULT_HOMOPHONES_CONFIDENCE_THRESHOLD
|
||||||
assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
|
assert config.spoken_word_confidence_threshold == DEFAULT_SPOKEN_WORD_CONFIDENCE_THRESHOLD
|
||||||
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
||||||
@@ -58,17 +60,20 @@ def test_threshold_overrides_take_precedence():
|
|||||||
config = AuditaConfig.from_sources(
|
config = AuditaConfig.from_sources(
|
||||||
env={
|
env={
|
||||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
|
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.6",
|
||||||
|
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.65",
|
||||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
|
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD": "0.7",
|
||||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75",
|
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD": "0.75",
|
||||||
},
|
},
|
||||||
overrides=ConfigOverrides(
|
overrides=ConfigOverrides(
|
||||||
glossary_confidence_threshold=0.85,
|
glossary_confidence_threshold=0.85,
|
||||||
|
grammar_confidence_threshold=0.88,
|
||||||
homophones_confidence_threshold=0.9,
|
homophones_confidence_threshold=0.9,
|
||||||
spoken_word_confidence_threshold=0.95,
|
spoken_word_confidence_threshold=0.95,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert config.glossary_confidence_threshold == 0.85
|
assert config.glossary_confidence_threshold == 0.85
|
||||||
|
assert config.grammar_confidence_threshold == 0.88
|
||||||
assert config.homophones_confidence_threshold == 0.9
|
assert config.homophones_confidence_threshold == 0.9
|
||||||
assert config.spoken_word_confidence_threshold == 0.95
|
assert config.spoken_word_confidence_threshold == 0.95
|
||||||
|
|
||||||
@@ -90,6 +95,7 @@ def test_invalid_module_sequences_are_rejected(value):
|
|||||||
"env_name",
|
"env_name",
|
||||||
[
|
[
|
||||||
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
|
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD",
|
||||||
|
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD",
|
||||||
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
|
"AUDITA_HOMOPHONES_CONFIDENCE_THRESHOLD",
|
||||||
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
|
"AUDITA_SPOKEN_WORD_CONFIDENCE_THRESHOLD",
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
|
|||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
|
{"corrections": []},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
revised = process_transcript(
|
revised = process_transcript(
|
||||||
@@ -78,6 +79,7 @@ def test_process_transcript_runs_noop_framework(tmp_path):
|
|||||||
"homophones:proposal",
|
"homophones:proposal",
|
||||||
"glossary_2:proposal",
|
"glossary_2:proposal",
|
||||||
"spoken_word:proposal",
|
"spoken_word:proposal",
|
||||||
|
"grammar:proposal",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -93,7 +95,9 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
|
|||||||
max_retries=config.max_retries,
|
max_retries=config.max_retries,
|
||||||
max_section_tokens=config.max_section_tokens,
|
max_section_tokens=config.max_section_tokens,
|
||||||
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
||||||
@@ -108,6 +112,7 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
|
|||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
|
{"corrections": []},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
||||||
@@ -135,6 +140,12 @@ def test_process_transcript_result_writes_report_and_preserves_skips_per_policy(
|
|||||||
"spoken_word_review",
|
"spoken_word_review",
|
||||||
"meaning_reversal_review",
|
"meaning_reversal_review",
|
||||||
]
|
]
|
||||||
|
assert [validator["name"] for validator in result.report.modules[4].to_dict()["validators"]] == [
|
||||||
|
"proposal_confidence_guard",
|
||||||
|
"protected_glossary_guard",
|
||||||
|
"grammar_only_guard",
|
||||||
|
"meaning_reversal_review",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_external_report_can_be_written(tmp_path):
|
def test_external_report_can_be_written(tmp_path):
|
||||||
@@ -145,6 +156,7 @@ def test_external_report_can_be_written(tmp_path):
|
|||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
{"corrections": []},
|
{"corrections": []},
|
||||||
|
{"corrections": []},
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
result = process_transcript_result(_transcript(), _glossary(), config, llm_client=llm_client)
|
||||||
@@ -191,7 +203,12 @@ def test_default_module_specs_expose_final_validator_order():
|
|||||||
"spoken_word_review",
|
"spoken_word_review",
|
||||||
"meaning_reversal_review",
|
"meaning_reversal_review",
|
||||||
]
|
]
|
||||||
assert [validator.name for validator in specs[4].module.validators()] == ["protected_glossary_guard"]
|
assert [validator.name for validator in specs[4].module.validators()] == [
|
||||||
|
"proposal_confidence_guard",
|
||||||
|
"protected_glossary_guard",
|
||||||
|
"grammar_only_guard",
|
||||||
|
"meaning_reversal_review",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
|
def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path):
|
||||||
@@ -206,7 +223,9 @@ def test_process_transcript_result_missing_api_key_writes_failed_report(tmp_path
|
|||||||
max_retries=config.max_retries,
|
max_retries=config.max_retries,
|
||||||
max_section_tokens=config.max_section_tokens,
|
max_section_tokens=config.max_section_tokens,
|
||||||
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
||||||
@@ -257,7 +276,9 @@ def test_process_transcript_result_preserves_partial_progress_when_later_module_
|
|||||||
max_retries=config.max_retries,
|
max_retries=config.max_retries,
|
||||||
max_section_tokens=config.max_section_tokens,
|
max_section_tokens=config.max_section_tokens,
|
||||||
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
glossary_confidence_threshold=config.glossary_confidence_threshold,
|
||||||
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
||||||
@@ -330,7 +351,9 @@ def test_process_transcript_result_preserves_partial_skips_and_validator_diagnos
|
|||||||
max_retries=config.max_retries,
|
max_retries=config.max_retries,
|
||||||
max_section_tokens=config.max_section_tokens,
|
max_section_tokens=config.max_section_tokens,
|
||||||
glossary_confidence_threshold=0.8,
|
glossary_confidence_threshold=0.8,
|
||||||
|
grammar_confidence_threshold=config.grammar_confidence_threshold,
|
||||||
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
homophones_confidence_threshold=config.homophones_confidence_threshold,
|
||||||
|
spoken_word_confidence_threshold=config.spoken_word_confidence_threshold,
|
||||||
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
normalize_max_segment_gap=config.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=config.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
normalize_max_segment_duration=config.normalize_max_segment_duration,
|
||||||
@@ -404,7 +427,7 @@ def test_process_transcript_result_supports_grammar_only_module_override(tmp_pat
|
|||||||
_glossary(),
|
_glossary(),
|
||||||
config,
|
config,
|
||||||
module_keys=["grammar"],
|
module_keys=["grammar"],
|
||||||
llm_client=FakeStructuredLLMClient([]),
|
llm_client=FakeStructuredLLMClient([{"corrections": []}]),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert [segment.id for segment in result.transcript] == [1, 2]
|
assert [segment.id for segment in result.transcript] == [1, 2]
|
||||||
|
|||||||
Reference in New Issue
Block a user