Implemented a second-stage validator for grammar corrections that allows homophone-related changes
This commit is contained in:
@@ -56,6 +56,11 @@ def _build_parser() -> argparse.ArgumentParser:
|
|||||||
type=float,
|
type=float,
|
||||||
help="minimum validator confidence required to apply a validated grammar correction",
|
help="minimum validator confidence required to apply a validated grammar correction",
|
||||||
)
|
)
|
||||||
|
process.add_argument(
|
||||||
|
"--grammar-spoken-form-validation-confidence-threshold",
|
||||||
|
type=float,
|
||||||
|
help="minimum validator confidence required to apply a spoken-form rescue correction",
|
||||||
|
)
|
||||||
process.add_argument(
|
process.add_argument(
|
||||||
"--normalize-max-segment-gap",
|
"--normalize-max-segment-gap",
|
||||||
type=float,
|
type=float,
|
||||||
@@ -94,6 +99,9 @@ def _process(args: argparse.Namespace) -> int:
|
|||||||
grammar_max_llm_passes=args.grammar_max_llm_passes,
|
grammar_max_llm_passes=args.grammar_max_llm_passes,
|
||||||
grammar_validation_enabled=args.grammar_validation_enabled,
|
grammar_validation_enabled=args.grammar_validation_enabled,
|
||||||
grammar_validation_confidence_threshold=args.grammar_validation_confidence_threshold,
|
grammar_validation_confidence_threshold=args.grammar_validation_confidence_threshold,
|
||||||
|
grammar_spoken_form_validation_confidence_threshold=(
|
||||||
|
args.grammar_spoken_form_validation_confidence_threshold
|
||||||
|
),
|
||||||
normalize_max_segment_gap=args.normalize_max_segment_gap,
|
normalize_max_segment_gap=args.normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=args.normalize_ellipsis_gap,
|
normalize_ellipsis_gap=args.normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=args.normalize_max_segment_duration,
|
normalize_max_segment_duration=args.normalize_max_segment_duration,
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3
|
|||||||
DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3
|
DEFAULT_GRAMMAR_MAX_LLM_PASSES = 3
|
||||||
DEFAULT_GRAMMAR_VALIDATION_ENABLED = True
|
DEFAULT_GRAMMAR_VALIDATION_ENABLED = True
|
||||||
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD = 0.80
|
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD = 0.80
|
||||||
|
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD = 0.80
|
||||||
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 5.0
|
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP = 5.0
|
||||||
DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5
|
DEFAULT_NORMALIZE_ELLIPSIS_GAP = 3.5
|
||||||
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0
|
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION = 60.0
|
||||||
@@ -36,6 +37,7 @@ class ConfigOverrides:
|
|||||||
grammar_max_llm_passes: Optional[int] = None
|
grammar_max_llm_passes: Optional[int] = None
|
||||||
grammar_validation_enabled: Optional[bool] = None
|
grammar_validation_enabled: Optional[bool] = None
|
||||||
grammar_validation_confidence_threshold: Optional[float] = None
|
grammar_validation_confidence_threshold: Optional[float] = None
|
||||||
|
grammar_spoken_form_validation_confidence_threshold: Optional[float] = None
|
||||||
normalize_max_segment_gap: Optional[float] = None
|
normalize_max_segment_gap: Optional[float] = None
|
||||||
normalize_ellipsis_gap: Optional[float] = None
|
normalize_ellipsis_gap: Optional[float] = None
|
||||||
normalize_max_segment_duration: Optional[float] = None
|
normalize_max_segment_duration: Optional[float] = None
|
||||||
@@ -56,6 +58,9 @@ class AuditaConfig:
|
|||||||
grammar_max_llm_passes: int = DEFAULT_GRAMMAR_MAX_LLM_PASSES
|
grammar_max_llm_passes: int = DEFAULT_GRAMMAR_MAX_LLM_PASSES
|
||||||
grammar_validation_enabled: bool = DEFAULT_GRAMMAR_VALIDATION_ENABLED
|
grammar_validation_enabled: bool = DEFAULT_GRAMMAR_VALIDATION_ENABLED
|
||||||
grammar_validation_confidence_threshold: float = DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
|
grammar_validation_confidence_threshold: float = DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
|
||||||
|
grammar_spoken_form_validation_confidence_threshold: float = (
|
||||||
|
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
|
||||||
|
)
|
||||||
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
normalize_max_segment_gap: float = DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
||||||
normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP
|
normalize_ellipsis_gap: float = DEFAULT_NORMALIZE_ELLIPSIS_GAP
|
||||||
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
|
normalize_max_segment_duration: float = DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
|
||||||
@@ -122,6 +127,12 @@ class AuditaConfig:
|
|||||||
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
|
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
|
||||||
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD",
|
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD",
|
||||||
)
|
)
|
||||||
|
grammar_spoken_form_validation_confidence_threshold = _select_float(
|
||||||
|
selected.grammar_spoken_form_validation_confidence_threshold,
|
||||||
|
source.get("AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD"),
|
||||||
|
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
|
||||||
|
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD",
|
||||||
|
)
|
||||||
normalize_max_segment_gap = _select_float(
|
normalize_max_segment_gap = _select_float(
|
||||||
selected.normalize_max_segment_gap,
|
selected.normalize_max_segment_gap,
|
||||||
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"),
|
source.get("AUDITA_NORMALIZE_MAX_SEGMENT_GAP"),
|
||||||
@@ -160,6 +171,7 @@ class AuditaConfig:
|
|||||||
grammar_max_llm_passes=grammar_max_llm_passes,
|
grammar_max_llm_passes=grammar_max_llm_passes,
|
||||||
grammar_validation_enabled=grammar_validation_enabled,
|
grammar_validation_enabled=grammar_validation_enabled,
|
||||||
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
|
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
|
||||||
|
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
|
||||||
normalize_max_segment_gap=normalize_max_segment_gap,
|
normalize_max_segment_gap=normalize_max_segment_gap,
|
||||||
normalize_ellipsis_gap=normalize_ellipsis_gap,
|
normalize_ellipsis_gap=normalize_ellipsis_gap,
|
||||||
normalize_max_segment_duration=normalize_max_segment_duration,
|
normalize_max_segment_duration=normalize_max_segment_duration,
|
||||||
@@ -192,6 +204,10 @@ class AuditaConfig:
|
|||||||
raise AuditaConfigError(
|
raise AuditaConfigError(
|
||||||
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0."
|
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0."
|
||||||
)
|
)
|
||||||
|
if not 0.0 <= self.grammar_spoken_form_validation_confidence_threshold <= 1.0:
|
||||||
|
raise AuditaConfigError(
|
||||||
|
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD must be between 0.0 and 1.0."
|
||||||
|
)
|
||||||
if not math.isfinite(self.normalize_max_segment_gap):
|
if not math.isfinite(self.normalize_max_segment_gap):
|
||||||
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.")
|
raise AuditaConfigError("AUDITA_NORMALIZE_MAX_SEGMENT_GAP must be finite.")
|
||||||
if not math.isfinite(self.normalize_ellipsis_gap):
|
if not math.isfinite(self.normalize_ellipsis_gap):
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ from typing import List
|
|||||||
from .config import AuditaConfig
|
from .config import AuditaConfig
|
||||||
from .errors import AuditaLLMError
|
from .errors import AuditaLLMError
|
||||||
from .prompts import Message
|
from .prompts import Message
|
||||||
from .schemas import CorrectionSet, GrammarValidationSet
|
from .schemas import CorrectionSet, GrammarSpokenFormValidationSet, GrammarValidationSet
|
||||||
|
|
||||||
|
|
||||||
class InstructorLLMClient:
|
class InstructorLLMClient:
|
||||||
@@ -52,6 +52,26 @@ class InstructorLLMClient:
|
|||||||
"supports tool calling or structured outputs."
|
"supports tool calling or structured outputs."
|
||||||
) from exc
|
) from exc
|
||||||
|
|
||||||
|
def create_grammar_spoken_form_validations(
|
||||||
|
self,
|
||||||
|
messages: List[Message],
|
||||||
|
config: AuditaConfig,
|
||||||
|
) -> GrammarSpokenFormValidationSet:
|
||||||
|
model = _normalize_openrouter_model(config.model)
|
||||||
|
try:
|
||||||
|
return self._client.chat.completions.create(
|
||||||
|
model=model,
|
||||||
|
messages=messages,
|
||||||
|
response_model=GrammarSpokenFormValidationSet,
|
||||||
|
max_retries=config.max_retries,
|
||||||
|
extra_body={"provider": {"require_parameters": True}},
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
raise AuditaLLMError(
|
||||||
|
"LLM spoken-form grammar validation request failed. Confirm the configured OpenRouter model "
|
||||||
|
"supports tool calling or structured outputs."
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
def _normalize_openrouter_model(model: str) -> str:
|
def _normalize_openrouter_model(model: str) -> str:
|
||||||
prefix = "openrouter/"
|
prefix = "openrouter/"
|
||||||
|
|||||||
@@ -5,7 +5,13 @@ from typing import List, Protocol
|
|||||||
from .chunking import TranscriptSection
|
from .chunking import TranscriptSection
|
||||||
from .config import AuditaConfig
|
from .config import AuditaConfig
|
||||||
from .prompts import build_glossary_correction_messages, build_grammar_correction_messages
|
from .prompts import build_glossary_correction_messages, build_grammar_correction_messages
|
||||||
from .schemas import CorrectionCandidate, CorrectionSet, Glossary, GrammarValidationSet
|
from .schemas import (
|
||||||
|
CorrectionCandidate,
|
||||||
|
CorrectionSet,
|
||||||
|
Glossary,
|
||||||
|
GrammarSpokenFormValidationSet,
|
||||||
|
GrammarValidationSet,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class LLMClient(Protocol):
|
class LLMClient(Protocol):
|
||||||
@@ -15,6 +21,13 @@ class LLMClient(Protocol):
|
|||||||
def create_grammar_validations(self, messages: List[dict], config: AuditaConfig) -> GrammarValidationSet:
|
def create_grammar_validations(self, messages: List[dict], config: AuditaConfig) -> GrammarValidationSet:
|
||||||
...
|
...
|
||||||
|
|
||||||
|
def create_grammar_spoken_form_validations(
|
||||||
|
self,
|
||||||
|
messages: List[dict],
|
||||||
|
config: AuditaConfig,
|
||||||
|
) -> GrammarSpokenFormValidationSet:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class CorrectionPass(Protocol):
|
class CorrectionPass(Protocol):
|
||||||
def run(
|
def run(
|
||||||
|
|||||||
@@ -12,10 +12,12 @@ from .corrections import CorrectionGuard, ReplacementMode, SkippedCorrection, ap
|
|||||||
from .errors import AuditaError
|
from .errors import AuditaError
|
||||||
from .normalization import NormalizationResult, normalize_transcript
|
from .normalization import NormalizationResult, normalize_transcript
|
||||||
from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient
|
from .passes import CorrectionPass, GlossaryCorrectionPass, GrammarCorrectionPass, LLMClient
|
||||||
from .prompts import build_grammar_validation_messages
|
from .prompts import build_grammar_spoken_form_validation_messages, build_grammar_validation_messages
|
||||||
from .protection import ProtectedVocabulary
|
from .protection import ProtectedVocabulary
|
||||||
from .semantic_validation import (
|
from .semantic_validation import (
|
||||||
filter_with_grammar_validations,
|
filter_with_meaning_preserving_validations,
|
||||||
|
filter_with_spoken_form_validations,
|
||||||
|
keep_corrections_with_indexes,
|
||||||
select_grammar_validation_candidates,
|
select_grammar_validation_candidates,
|
||||||
)
|
)
|
||||||
from .schemas import CorrectionCandidate, Glossary, SourceTranscriptSegment, TranscriptSegment, parse_transcript_json
|
from .schemas import CorrectionCandidate, Glossary, SourceTranscriptSegment, TranscriptSegment, parse_transcript_json
|
||||||
@@ -266,6 +268,9 @@ def _validate_grammar_corrections(
|
|||||||
"validation_approved_count": 0,
|
"validation_approved_count": 0,
|
||||||
"validation_rejected_count": 0,
|
"validation_rejected_count": 0,
|
||||||
"validation_bypassed_count": 0,
|
"validation_bypassed_count": 0,
|
||||||
|
"spoken_form_validation_candidate_count": 0,
|
||||||
|
"spoken_form_validation_approved_count": 0,
|
||||||
|
"spoken_form_validation_rejected_count": 0,
|
||||||
}
|
}
|
||||||
if stage.name != "grammar":
|
if stage.name != "grammar":
|
||||||
return corrections, [], validation_summary
|
return corrections, [], validation_summary
|
||||||
@@ -297,16 +302,47 @@ def _validate_grammar_corrections(
|
|||||||
response_path = pass_dir / "validation-response-0000.json"
|
response_path = pass_dir / "validation-response-0000.json"
|
||||||
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
response_path.write_text(response.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
result = filter_with_grammar_validations(
|
meaning_result = filter_with_meaning_preserving_validations(
|
||||||
corrections,
|
|
||||||
candidates,
|
candidates,
|
||||||
response,
|
response,
|
||||||
config.grammar_validation_confidence_threshold,
|
config.grammar_validation_confidence_threshold,
|
||||||
)
|
)
|
||||||
validation_summary["validation_approved_count"] = result.approved_count
|
validation_summary["validation_approved_count"] = meaning_result.approved_count
|
||||||
validation_summary["validation_rejected_count"] = result.rejected_count
|
validation_summary["validation_rejected_count"] = meaning_result.rejected_count
|
||||||
validation_summary["validation_bypassed_count"] = result.bypassed_count
|
validation_summary["spoken_form_validation_candidate_count"] = len(meaning_result.rescue_candidates)
|
||||||
return result.corrections, result.skipped, validation_summary
|
|
||||||
|
candidate_indexes = {candidate.correction_index for candidate in candidates}
|
||||||
|
allowed_indexes = set(range(len(corrections))) - candidate_indexes
|
||||||
|
allowed_indexes.update(meaning_result.approved_correction_indexes)
|
||||||
|
|
||||||
|
if not meaning_result.rescue_candidates:
|
||||||
|
return keep_corrections_with_indexes(corrections, allowed_indexes), [], validation_summary
|
||||||
|
|
||||||
|
spoken_form_payload = [candidate.to_prompt_payload() for candidate in meaning_result.rescue_candidates]
|
||||||
|
spoken_form_messages = build_grammar_spoken_form_validation_messages(spoken_form_payload)
|
||||||
|
spoken_form_prompt_path = pass_dir / "spoken-form-validation-prompt-0000.json"
|
||||||
|
spoken_form_prompt_path.write_text(
|
||||||
|
json.dumps(spoken_form_messages, ensure_ascii=False, indent=2) + "\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
spoken_form_response = llm_client.create_grammar_spoken_form_validations(spoken_form_messages, config)
|
||||||
|
spoken_form_response_path = pass_dir / "spoken-form-validation-response-0000.json"
|
||||||
|
spoken_form_response_path.write_text(spoken_form_response.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
spoken_form_result = filter_with_spoken_form_validations(
|
||||||
|
meaning_result.rescue_candidates,
|
||||||
|
spoken_form_response,
|
||||||
|
config.grammar_spoken_form_validation_confidence_threshold,
|
||||||
|
)
|
||||||
|
validation_summary["spoken_form_validation_approved_count"] = spoken_form_result.approved_count
|
||||||
|
validation_summary["spoken_form_validation_rejected_count"] = spoken_form_result.rejected_count
|
||||||
|
allowed_indexes.update(spoken_form_result.approved_correction_indexes)
|
||||||
|
return (
|
||||||
|
keep_corrections_with_indexes(corrections, allowed_indexes),
|
||||||
|
spoken_form_result.skipped,
|
||||||
|
validation_summary,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _create_run_dir(work_dir: Path) -> Path:
|
def _create_run_dir(work_dir: Path) -> Path:
|
||||||
@@ -331,6 +367,9 @@ def _write_run_metadata(
|
|||||||
"grammar_confidence_threshold": config.grammar_confidence_threshold,
|
"grammar_confidence_threshold": config.grammar_confidence_threshold,
|
||||||
"grammar_validation_enabled": config.grammar_validation_enabled,
|
"grammar_validation_enabled": config.grammar_validation_enabled,
|
||||||
"grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold,
|
"grammar_validation_confidence_threshold": config.grammar_validation_confidence_threshold,
|
||||||
|
"grammar_spoken_form_validation_confidence_threshold": (
|
||||||
|
config.grammar_spoken_form_validation_confidence_threshold
|
||||||
|
),
|
||||||
"max_retries": config.max_retries,
|
"max_retries": config.max_retries,
|
||||||
"glossary_max_llm_passes": config.glossary_max_llm_passes,
|
"glossary_max_llm_passes": config.glossary_max_llm_passes,
|
||||||
"grammar_max_llm_passes": config.grammar_max_llm_passes,
|
"grammar_max_llm_passes": config.grammar_max_llm_passes,
|
||||||
|
|||||||
@@ -116,7 +116,8 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me
|
|||||||
system = (
|
system = (
|
||||||
"You are Audita, a conservative semantic validation assistant. "
|
"You are Audita, a conservative semantic validation assistant. "
|
||||||
"Evaluate whether each proposed grammar correction preserves the same spoken content and meaning. "
|
"Evaluate whether each proposed grammar correction preserves the same spoken content and meaning. "
|
||||||
"Do not judge whether the correction is more polished; judge only whether it changes meaning."
|
"Do not judge whether the correction is more polished, and do not try to rescue likely homophone or transcription fixes. "
|
||||||
|
"Judge only whether the corrected text preserves the same written meaning."
|
||||||
)
|
)
|
||||||
user = (
|
user = (
|
||||||
"Review these proposed grammar corrections and decide whether each correction preserves meaning.\n\n"
|
"Review these proposed grammar corrections and decide whether each correction preserves meaning.\n\n"
|
||||||
@@ -125,10 +126,34 @@ def build_grammar_validation_messages(validation_payload: List[dict]) -> List[Me
|
|||||||
"- Reject corrections that add or remove negation, reverse meaning, introduce antonyms, change names, change quantities, change actions, change who did what, or otherwise substantively alter the speaker's meaning.\n"
|
"- Reject corrections that add or remove negation, reverse meaning, introduce antonyms, change names, change quantities, change actions, change who did what, or otherwise substantively alter the speaker's meaning.\n"
|
||||||
"- Reject corrections like changing \"became visible\" to \"became invisible\" because that reverses the meaning.\n"
|
"- Reject corrections like changing \"became visible\" to \"became invisible\" because that reverses the meaning.\n"
|
||||||
"- Allow capitalization and punctuation changes when they preserve meaning.\n"
|
"- Allow capitalization and punctuation changes when they preserve meaning.\n"
|
||||||
"- Allow spelling and homophone fixes only when the corrected segment preserves the same spoken content.\n"
|
"- If a correction changes meaning because the original transcript may have used the wrong homophone or a phonetic misspelling, reject it here; that question is handled in a separate spoken-form validation step.\n"
|
||||||
"- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n"
|
"- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n"
|
||||||
"- Each returned validation must contain only correction_index, is_meaning_preserving, confidence, and reason.\n"
|
"- Each returned validation must contain only correction_index, is_meaning_preserving, confidence, and reason.\n"
|
||||||
"- confidence must be between 0.0 and 1.0.\n\n"
|
"- confidence must be between 0.0 and 1.0.\n\n"
|
||||||
f"Corrections to validate:\n{payload_json}"
|
f"Corrections to validate:\n{payload_json}"
|
||||||
)
|
)
|
||||||
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
||||||
|
|
||||||
|
|
||||||
|
def build_grammar_spoken_form_validation_messages(validation_payload: List[dict]) -> List[Message]:
|
||||||
|
payload_json = json.dumps(validation_payload, ensure_ascii=False, indent=2)
|
||||||
|
system = (
|
||||||
|
"You are Audita, a conservative spoken-form validation assistant. "
|
||||||
|
"Evaluate whether each rejected grammar correction is still a likely homophone, spoken-form, or transcription fix "
|
||||||
|
"supported by the local segment context. "
|
||||||
|
"Approve only corrections that plausibly recover the intended spoken words from a mistaken transcript rendering."
|
||||||
|
)
|
||||||
|
user = (
|
||||||
|
"Review these rejected grammar corrections and decide whether each one is a likely spoken-form correction.\n\n"
|
||||||
|
"Rules:\n"
|
||||||
|
"- Return one validation decision for every correction_index in the input.\n"
|
||||||
|
"- Approve a correction only when the original transcript text is plausibly a mistaken homophone, phonetic rendering, or transcription error, and the corrected text better matches the likely spoken words in context.\n"
|
||||||
|
"- Allow examples like changing \"dam\" to \"damn\" when the surrounding phrase strongly supports the intended spoken phrase.\n"
|
||||||
|
"- Reject examples like changing \"became visible\" to \"became invisible\" because that is a semantic reversal, not a likely spoken-form correction.\n"
|
||||||
|
"- Reject paraphrases, stylistic rewrites, content additions or removals, and any meaning-changing edit that is not clearly explained by a transcription or spoken-form mistake.\n"
|
||||||
|
"- Do not use domain knowledge to second-guess protected glossary terms; glossary-protected corrections are excluded from this validation step.\n"
|
||||||
|
"- Each returned validation must contain only correction_index, is_likely_spoken_form_correction, confidence, and reason.\n"
|
||||||
|
"- confidence must be between 0.0 and 1.0.\n\n"
|
||||||
|
f"Corrections to validate:\n{payload_json}"
|
||||||
|
)
|
||||||
|
return [{"role": "system", "content": system}, {"role": "user", "content": user}]
|
||||||
|
|||||||
@@ -217,6 +217,52 @@ class GrammarValidationSet(BaseModel):
|
|||||||
validations: List[GrammarValidationDecision] = Field(default_factory=list)
|
validations: List[GrammarValidationDecision] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
class GrammarSpokenFormValidationDecision(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
correction_index: int = Field(ge=0)
|
||||||
|
is_likely_spoken_form_correction: bool
|
||||||
|
confidence: float = Field(ge=0.0, le=1.0)
|
||||||
|
reason: StrictStr
|
||||||
|
|
||||||
|
@field_validator("correction_index", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def require_integer_index(cls, value: Any) -> int:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int):
|
||||||
|
raise ValueError("must be an integer")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("is_likely_spoken_form_correction", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def require_boolean(cls, value: Any) -> bool:
|
||||||
|
if not isinstance(value, bool):
|
||||||
|
raise ValueError("must be a boolean")
|
||||||
|
return value
|
||||||
|
|
||||||
|
@field_validator("confidence", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def require_number(cls, value: Any) -> float:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||||
|
raise ValueError("must be a JSON number")
|
||||||
|
number = float(value)
|
||||||
|
if not math.isfinite(number):
|
||||||
|
raise ValueError("must be finite")
|
||||||
|
return number
|
||||||
|
|
||||||
|
@field_validator("reason")
|
||||||
|
@classmethod
|
||||||
|
def require_non_empty_reason(cls, value: str) -> str:
|
||||||
|
if not value.strip():
|
||||||
|
raise ValueError("must not be empty")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class GrammarSpokenFormValidationSet(BaseModel):
|
||||||
|
model_config = ConfigDict(extra="forbid")
|
||||||
|
|
||||||
|
validations: List[GrammarSpokenFormValidationDecision] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment])
|
_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment])
|
||||||
_SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment])
|
_SOURCE_TRANSCRIPT_ADAPTER = TypeAdapter(List[SourceTranscriptSegment])
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
import string
|
import string
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Dict, List, Optional, Tuple
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||||
|
|
||||||
from .corrections import ReplacementMode, SkippedCorrection
|
from .corrections import ReplacementMode, SkippedCorrection
|
||||||
from .errors import AuditaLLMError
|
from .errors import AuditaLLMError
|
||||||
from .protection import ProtectedVocabulary
|
from .protection import ProtectedVocabulary
|
||||||
from .schemas import CorrectionCandidate, GrammarValidationSet, TranscriptSegment
|
from .schemas import (
|
||||||
|
CorrectionCandidate,
|
||||||
|
GrammarSpokenFormValidationSet,
|
||||||
|
GrammarValidationSet,
|
||||||
|
TranscriptSegment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -27,13 +32,19 @@ class GrammarValidationCandidate:
|
|||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class GrammarValidationFilterResult:
|
class GrammarMeaningValidationResult:
|
||||||
corrections: List[CorrectionCandidate]
|
approved_correction_indexes: List[int]
|
||||||
skipped: List[SkippedCorrection]
|
rescue_candidates: List[GrammarValidationCandidate]
|
||||||
candidate_count: int
|
approved_count: int
|
||||||
|
rejected_count: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class GrammarSpokenFormValidationResult:
|
||||||
|
approved_correction_indexes: List[int]
|
||||||
|
skipped: List[SkippedCorrection]
|
||||||
approved_count: int
|
approved_count: int
|
||||||
rejected_count: int
|
rejected_count: int
|
||||||
bypassed_count: int
|
|
||||||
|
|
||||||
|
|
||||||
def select_grammar_validation_candidates(
|
def select_grammar_validation_candidates(
|
||||||
@@ -74,30 +85,58 @@ def select_grammar_validation_candidates(
|
|||||||
return candidates, bypassed_count
|
return candidates, bypassed_count
|
||||||
|
|
||||||
|
|
||||||
def filter_with_grammar_validations(
|
def filter_with_meaning_preserving_validations(
|
||||||
corrections: List[CorrectionCandidate],
|
|
||||||
candidates: List[GrammarValidationCandidate],
|
candidates: List[GrammarValidationCandidate],
|
||||||
validation_set: GrammarValidationSet,
|
validation_set: GrammarValidationSet,
|
||||||
confidence_threshold: float,
|
confidence_threshold: float,
|
||||||
) -> GrammarValidationFilterResult:
|
) -> GrammarMeaningValidationResult:
|
||||||
candidate_indexes = {candidate.correction_index for candidate in candidates}
|
decisions_by_index = _index_validation_decisions(
|
||||||
decisions_by_index = {}
|
validation_set.validations,
|
||||||
for decision in validation_set.validations:
|
candidates,
|
||||||
if decision.correction_index in decisions_by_index:
|
duplicate_error="LLM grammar validation response included duplicate correction_index values.",
|
||||||
raise AuditaLLMError("LLM grammar validation response included duplicate correction_index values.")
|
unknown_error="LLM grammar validation response included an unknown correction_index.",
|
||||||
if decision.correction_index not in candidate_indexes:
|
missing_error="LLM grammar validation response omitted correction_index values.",
|
||||||
raise AuditaLLMError("LLM grammar validation response included an unknown correction_index.")
|
)
|
||||||
decisions_by_index[decision.correction_index] = decision
|
|
||||||
|
|
||||||
missing_indexes = sorted(candidate_indexes - set(decisions_by_index))
|
approved_correction_indexes: List[int] = []
|
||||||
if missing_indexes:
|
rescue_candidates: List[GrammarValidationCandidate] = []
|
||||||
raise AuditaLLMError("LLM grammar validation response omitted correction_index values.")
|
|
||||||
|
|
||||||
rejected_by_index: Dict[int, SkippedCorrection] = {}
|
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
decision = decisions_by_index[candidate.correction_index]
|
decision = decisions_by_index[candidate.correction_index]
|
||||||
if not decision.is_meaning_preserving or decision.confidence < confidence_threshold:
|
if decision.is_meaning_preserving and decision.confidence >= confidence_threshold:
|
||||||
rejected_by_index[candidate.correction_index] = SkippedCorrection(
|
approved_correction_indexes.append(candidate.correction_index)
|
||||||
|
else:
|
||||||
|
rescue_candidates.append(candidate)
|
||||||
|
|
||||||
|
return GrammarMeaningValidationResult(
|
||||||
|
approved_correction_indexes=approved_correction_indexes,
|
||||||
|
rescue_candidates=rescue_candidates,
|
||||||
|
approved_count=len(approved_correction_indexes),
|
||||||
|
rejected_count=len(rescue_candidates),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def filter_with_spoken_form_validations(
|
||||||
|
candidates: List[GrammarValidationCandidate],
|
||||||
|
validation_set: GrammarSpokenFormValidationSet,
|
||||||
|
confidence_threshold: float,
|
||||||
|
) -> GrammarSpokenFormValidationResult:
|
||||||
|
decisions_by_index = _index_validation_decisions(
|
||||||
|
validation_set.validations,
|
||||||
|
candidates,
|
||||||
|
duplicate_error="LLM spoken-form validation response included duplicate correction_index values.",
|
||||||
|
unknown_error="LLM spoken-form validation response included an unknown correction_index.",
|
||||||
|
missing_error="LLM spoken-form validation response omitted correction_index values.",
|
||||||
|
)
|
||||||
|
|
||||||
|
approved_correction_indexes: List[int] = []
|
||||||
|
skipped: List[SkippedCorrection] = []
|
||||||
|
for candidate in candidates:
|
||||||
|
decision = decisions_by_index[candidate.correction_index]
|
||||||
|
if decision.is_likely_spoken_form_correction and decision.confidence >= confidence_threshold:
|
||||||
|
approved_correction_indexes.append(candidate.correction_index)
|
||||||
|
continue
|
||||||
|
skipped.append(
|
||||||
|
SkippedCorrection(
|
||||||
id=candidate.correction.id,
|
id=candidate.correction.id,
|
||||||
reason="grammar validation rejected semantic change",
|
reason="grammar validation rejected semantic change",
|
||||||
original_text=candidate.correction.original_text,
|
original_text=candidate.correction.original_text,
|
||||||
@@ -107,21 +146,24 @@ def filter_with_grammar_validations(
|
|||||||
validation_confidence=decision.confidence,
|
validation_confidence=decision.confidence,
|
||||||
validation_reason=decision.reason,
|
validation_reason=decision.reason,
|
||||||
)
|
)
|
||||||
|
)
|
||||||
|
|
||||||
filtered_corrections = [
|
return GrammarSpokenFormValidationResult(
|
||||||
correction for index, correction in enumerate(corrections) if index not in rejected_by_index
|
approved_correction_indexes=approved_correction_indexes,
|
||||||
]
|
skipped=skipped,
|
||||||
approved_count = len(candidates) - len(rejected_by_index)
|
approved_count=len(approved_correction_indexes),
|
||||||
return GrammarValidationFilterResult(
|
rejected_count=len(skipped),
|
||||||
corrections=filtered_corrections,
|
|
||||||
skipped=[rejected_by_index[index] for index in sorted(rejected_by_index)],
|
|
||||||
candidate_count=len(candidates),
|
|
||||||
approved_count=approved_count,
|
|
||||||
rejected_count=len(rejected_by_index),
|
|
||||||
bypassed_count=len(corrections) - len(candidates),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def keep_corrections_with_indexes(
|
||||||
|
corrections: List[CorrectionCandidate],
|
||||||
|
allowed_indexes: Iterable[int],
|
||||||
|
) -> List[CorrectionCandidate]:
|
||||||
|
allowed_index_set = set(allowed_indexes)
|
||||||
|
return [correction for index, correction in enumerate(corrections) if index in allowed_index_set]
|
||||||
|
|
||||||
|
|
||||||
def is_capitalization_or_punctuation_only(original_text: str, corrected_text: str) -> bool:
|
def is_capitalization_or_punctuation_only(original_text: str, corrected_text: str) -> bool:
|
||||||
return _semantic_key(original_text) == _semantic_key(corrected_text)
|
return _semantic_key(original_text) == _semantic_key(corrected_text)
|
||||||
|
|
||||||
@@ -152,6 +194,28 @@ def _preview_corrected_segment_text(
|
|||||||
return segment.text.replace(correction.original_text, correction.corrected_text)
|
return segment.text.replace(correction.original_text, correction.corrected_text)
|
||||||
|
|
||||||
|
|
||||||
|
def _index_validation_decisions(
|
||||||
|
decisions: Iterable[Any],
|
||||||
|
candidates: List[GrammarValidationCandidate],
|
||||||
|
duplicate_error: str,
|
||||||
|
unknown_error: str,
|
||||||
|
missing_error: str,
|
||||||
|
) -> Dict[int, Any]:
|
||||||
|
candidate_indexes = {candidate.correction_index for candidate in candidates}
|
||||||
|
decisions_by_index: Dict[int, Any] = {}
|
||||||
|
for decision in decisions:
|
||||||
|
if decision.correction_index in decisions_by_index:
|
||||||
|
raise AuditaLLMError(duplicate_error)
|
||||||
|
if decision.correction_index not in candidate_indexes:
|
||||||
|
raise AuditaLLMError(unknown_error)
|
||||||
|
decisions_by_index[decision.correction_index] = decision
|
||||||
|
|
||||||
|
missing_indexes = sorted(candidate_indexes - set(decisions_by_index))
|
||||||
|
if missing_indexes:
|
||||||
|
raise AuditaLLMError(missing_error)
|
||||||
|
return decisions_by_index
|
||||||
|
|
||||||
|
|
||||||
_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"}
|
_PUNCTUATION = set(string.punctuation) | {"—", "–", "…", "“", "”", "‘", "’"}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ def test_process_help_includes_glossary_pass_flag(capsys):
|
|||||||
assert "--grammar-confidence-threshold" in output
|
assert "--grammar-confidence-threshold" in output
|
||||||
assert "--grammar-validation-enabled" in output
|
assert "--grammar-validation-enabled" in output
|
||||||
assert "--grammar-validation-confidence-threshold" in output
|
assert "--grammar-validation-confidence-threshold" in output
|
||||||
|
assert "--grammar-spoken-form-validation-confidence-threshold" in output
|
||||||
assert "--normalize-max-segment-gap" in output
|
assert "--normalize-max-segment-gap" in output
|
||||||
assert "--normalize-ellipsis-gap" in output
|
assert "--normalize-ellipsis-gap" in output
|
||||||
assert "--normalize-max-segment-duration" in output
|
assert "--normalize-max-segment-duration" in output
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from audita.config import (
|
|||||||
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
|
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
|
||||||
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
|
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
|
||||||
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
|
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
|
||||||
|
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
|
||||||
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
|
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
|
||||||
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
|
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
|
||||||
DEFAULT_MAX_RETRIES,
|
DEFAULT_MAX_RETRIES,
|
||||||
@@ -36,6 +37,11 @@ def test_config_uses_defaults_with_api_key():
|
|||||||
assert config.grammar_validation_enabled is True
|
assert config.grammar_validation_enabled is True
|
||||||
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
|
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
|
||||||
assert config.grammar_validation_confidence_threshold == 0.8
|
assert config.grammar_validation_confidence_threshold == 0.8
|
||||||
|
assert (
|
||||||
|
config.grammar_spoken_form_validation_confidence_threshold
|
||||||
|
== DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
|
||||||
|
)
|
||||||
|
assert config.grammar_spoken_form_validation_confidence_threshold == 0.8
|
||||||
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
|
||||||
assert config.normalize_max_segment_gap == 5.0
|
assert config.normalize_max_segment_gap == 5.0
|
||||||
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
|
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
|
||||||
@@ -58,6 +64,7 @@ def test_config_env_overrides_defaults():
|
|||||||
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
|
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
|
||||||
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
|
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
|
||||||
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
|
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
|
||||||
|
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
|
||||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
|
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
|
||||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
|
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
|
||||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
|
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
|
||||||
@@ -74,6 +81,7 @@ def test_config_env_overrides_defaults():
|
|||||||
assert config.grammar_max_llm_passes == 4
|
assert config.grammar_max_llm_passes == 4
|
||||||
assert config.grammar_validation_enabled is False
|
assert config.grammar_validation_enabled is False
|
||||||
assert config.grammar_validation_confidence_threshold == 0.91
|
assert config.grammar_validation_confidence_threshold == 0.91
|
||||||
|
assert config.grammar_spoken_form_validation_confidence_threshold == 0.87
|
||||||
assert config.normalize_max_segment_gap == 4.5
|
assert config.normalize_max_segment_gap == 4.5
|
||||||
assert config.normalize_ellipsis_gap == 1.5
|
assert config.normalize_ellipsis_gap == 1.5
|
||||||
assert config.normalize_max_segment_duration == 45.0
|
assert config.normalize_max_segment_duration == 45.0
|
||||||
@@ -91,6 +99,7 @@ def test_config_cli_overrides_env():
|
|||||||
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
|
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
|
||||||
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
|
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
|
||||||
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
|
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
|
||||||
|
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
|
||||||
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
|
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
|
||||||
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
|
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
|
||||||
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
|
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
|
||||||
@@ -106,6 +115,7 @@ def test_config_cli_overrides_env():
|
|||||||
grammar_max_llm_passes=3,
|
grammar_max_llm_passes=3,
|
||||||
grammar_validation_enabled=True,
|
grammar_validation_enabled=True,
|
||||||
grammar_validation_confidence_threshold=0.75,
|
grammar_validation_confidence_threshold=0.75,
|
||||||
|
grammar_spoken_form_validation_confidence_threshold=0.72,
|
||||||
normalize_max_segment_gap=3.0,
|
normalize_max_segment_gap=3.0,
|
||||||
normalize_ellipsis_gap=1.0,
|
normalize_ellipsis_gap=1.0,
|
||||||
normalize_max_segment_duration=30.0,
|
normalize_max_segment_duration=30.0,
|
||||||
@@ -122,6 +132,7 @@ def test_config_cli_overrides_env():
|
|||||||
assert config.grammar_max_llm_passes == 3
|
assert config.grammar_max_llm_passes == 3
|
||||||
assert config.grammar_validation_enabled is True
|
assert config.grammar_validation_enabled is True
|
||||||
assert config.grammar_validation_confidence_threshold == 0.75
|
assert config.grammar_validation_confidence_threshold == 0.75
|
||||||
|
assert config.grammar_spoken_form_validation_confidence_threshold == 0.72
|
||||||
assert config.normalize_max_segment_gap == 3.0
|
assert config.normalize_max_segment_gap == 3.0
|
||||||
assert config.normalize_ellipsis_gap == 1.0
|
assert config.normalize_ellipsis_gap == 1.0
|
||||||
assert config.normalize_max_segment_duration == 30.0
|
assert config.normalize_max_segment_duration == 30.0
|
||||||
@@ -168,6 +179,13 @@ def test_config_rejects_invalid_stage_thresholds():
|
|||||||
AuditaConfig.from_sources(
|
AuditaConfig.from_sources(
|
||||||
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
|
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
|
||||||
)
|
)
|
||||||
|
with pytest.raises(AuditaConfigError):
|
||||||
|
AuditaConfig.from_sources(
|
||||||
|
env={
|
||||||
|
"OPENROUTER_API_KEY": "key",
|
||||||
|
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "-0.1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_config_rejects_invalid_grammar_validation_enabled():
|
def test_config_rejects_invalid_grammar_validation_enabled():
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ from audita.errors import AuditaError
|
|||||||
from audita.pipeline import process_transcript
|
from audita.pipeline import process_transcript
|
||||||
from audita.schemas import (
|
from audita.schemas import (
|
||||||
CorrectionCandidate,
|
CorrectionCandidate,
|
||||||
|
GrammarSpokenFormValidationDecision,
|
||||||
|
GrammarSpokenFormValidationSet,
|
||||||
CorrectionSet,
|
CorrectionSet,
|
||||||
GrammarValidationDecision,
|
GrammarValidationDecision,
|
||||||
GrammarValidationSet,
|
GrammarValidationSet,
|
||||||
@@ -16,13 +18,16 @@ from audita.schemas import (
|
|||||||
|
|
||||||
|
|
||||||
class FakeLLMClient:
|
class FakeLLMClient:
|
||||||
def __init__(self, responses, validation_responses=None):
|
def __init__(self, responses, validation_responses=None, spoken_form_validation_responses=None):
|
||||||
self.responses = list(responses)
|
self.responses = list(responses)
|
||||||
self.validation_responses = list(validation_responses or [])
|
self.validation_responses = list(validation_responses or [])
|
||||||
|
self.spoken_form_validation_responses = list(spoken_form_validation_responses or [])
|
||||||
self.calls = 0
|
self.calls = 0
|
||||||
self.validation_calls = 0
|
self.validation_calls = 0
|
||||||
|
self.spoken_form_validation_calls = 0
|
||||||
self.messages = []
|
self.messages = []
|
||||||
self.validation_messages = []
|
self.validation_messages = []
|
||||||
|
self.spoken_form_validation_messages = []
|
||||||
|
|
||||||
def create_corrections(self, messages, config):
|
def create_corrections(self, messages, config):
|
||||||
self.calls += 1
|
self.calls += 1
|
||||||
@@ -36,6 +41,13 @@ class FakeLLMClient:
|
|||||||
raise AssertionError("Unexpected grammar validation request.")
|
raise AssertionError("Unexpected grammar validation request.")
|
||||||
return self.validation_responses.pop(0)
|
return self.validation_responses.pop(0)
|
||||||
|
|
||||||
|
def create_grammar_spoken_form_validations(self, messages, config):
|
||||||
|
self.spoken_form_validation_calls += 1
|
||||||
|
self.spoken_form_validation_messages.append(messages)
|
||||||
|
if not self.spoken_form_validation_responses:
|
||||||
|
raise AssertionError("Unexpected spoken-form validation request.")
|
||||||
|
return self.spoken_form_validation_responses.pop(0)
|
||||||
|
|
||||||
|
|
||||||
def _config(
|
def _config(
|
||||||
tmp_path,
|
tmp_path,
|
||||||
@@ -43,6 +55,7 @@ def _config(
|
|||||||
grammar_max_llm_passes=3,
|
grammar_max_llm_passes=3,
|
||||||
grammar_validation_enabled=False,
|
grammar_validation_enabled=False,
|
||||||
grammar_validation_confidence_threshold=0.8,
|
grammar_validation_confidence_threshold=0.8,
|
||||||
|
grammar_spoken_form_validation_confidence_threshold=0.8,
|
||||||
):
|
):
|
||||||
return AuditaConfig(
|
return AuditaConfig(
|
||||||
api_key="key",
|
api_key="key",
|
||||||
@@ -54,6 +67,7 @@ def _config(
|
|||||||
grammar_max_llm_passes=grammar_max_llm_passes,
|
grammar_max_llm_passes=grammar_max_llm_passes,
|
||||||
grammar_validation_enabled=grammar_validation_enabled,
|
grammar_validation_enabled=grammar_validation_enabled,
|
||||||
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
|
grammar_validation_confidence_threshold=grammar_validation_confidence_threshold,
|
||||||
|
grammar_spoken_form_validation_confidence_threshold=grammar_spoken_form_validation_confidence_threshold,
|
||||||
work_dir=tmp_path / "work",
|
work_dir=tmp_path / "work",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -698,6 +712,7 @@ def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
|
|||||||
assert metadata["grammar_confidence_threshold"] == 0.8
|
assert metadata["grammar_confidence_threshold"] == 0.8
|
||||||
assert metadata["grammar_validation_enabled"] is False
|
assert metadata["grammar_validation_enabled"] is False
|
||||||
assert metadata["grammar_validation_confidence_threshold"] == 0.8
|
assert metadata["grammar_validation_confidence_threshold"] == 0.8
|
||||||
|
assert metadata["grammar_spoken_form_validation_confidence_threshold"] == 0.8
|
||||||
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
|
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
|
||||||
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
|
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
|
||||||
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
|
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
|
||||||
@@ -766,12 +781,21 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
|
|||||||
confidence=0.99,
|
confidence=0.99,
|
||||||
reason="This reverses visible to invisible.",
|
reason="This reverses visible to invisible.",
|
||||||
)
|
)
|
||||||
|
spoken_form_validation = GrammarSpokenFormValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_likely_spoken_form_correction=False,
|
||||||
|
confidence=0.98,
|
||||||
|
reason="This is a semantic reversal, not a likely spoken-form correction.",
|
||||||
|
)
|
||||||
fake_client = FakeLLMClient(
|
fake_client = FakeLLMClient(
|
||||||
[
|
[
|
||||||
CorrectionSet(corrections=[]),
|
CorrectionSet(corrections=[]),
|
||||||
CorrectionSet(corrections=[grammar_correction]),
|
CorrectionSet(corrections=[grammar_correction]),
|
||||||
],
|
],
|
||||||
validation_responses=[GrammarValidationSet(validations=[validation])],
|
validation_responses=[GrammarValidationSet(validations=[validation])],
|
||||||
|
spoken_form_validation_responses=[
|
||||||
|
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
revised = process_transcript(
|
revised = process_transcript(
|
||||||
@@ -783,20 +807,24 @@ def test_grammar_validation_rejects_semantic_change(tmp_path):
|
|||||||
|
|
||||||
assert revised[0].text == "He became visible."
|
assert revised[0].text == "He became visible."
|
||||||
assert fake_client.validation_calls == 1
|
assert fake_client.validation_calls == 1
|
||||||
|
assert fake_client.spoken_form_validation_calls == 1
|
||||||
run_dirs = list((tmp_path / "work").iterdir())
|
run_dirs = list((tmp_path / "work").iterdir())
|
||||||
assert len(run_dirs) == 1
|
assert len(run_dirs) == 1
|
||||||
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
|
||||||
skipped = diagnostics["skipped_corrections"][0]
|
skipped = diagnostics["skipped_corrections"][0]
|
||||||
assert skipped["stage"] == "grammar"
|
assert skipped["stage"] == "grammar"
|
||||||
assert skipped["reason"] == "grammar validation rejected semantic change"
|
assert skipped["reason"] == "grammar validation rejected semantic change"
|
||||||
assert skipped["validation_confidence"] == 0.99
|
assert skipped["validation_confidence"] == 0.98
|
||||||
assert skipped["validation_reason"] == "This reverses visible to invisible."
|
assert skipped["validation_reason"] == "This is a semantic reversal, not a likely spoken-form correction."
|
||||||
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
|
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
|
||||||
grammar_pass = metadata["stages"][1]["passes"][0]
|
grammar_pass = metadata["stages"][1]["passes"][0]
|
||||||
assert grammar_pass["validation_candidate_count"] == 1
|
assert grammar_pass["validation_candidate_count"] == 1
|
||||||
assert grammar_pass["validation_approved_count"] == 0
|
assert grammar_pass["validation_approved_count"] == 0
|
||||||
assert grammar_pass["validation_rejected_count"] == 1
|
assert grammar_pass["validation_rejected_count"] == 1
|
||||||
assert grammar_pass["validation_bypassed_count"] == 0
|
assert grammar_pass["validation_bypassed_count"] == 0
|
||||||
|
assert grammar_pass["spoken_form_validation_candidate_count"] == 1
|
||||||
|
assert grammar_pass["spoken_form_validation_approved_count"] == 0
|
||||||
|
assert grammar_pass["spoken_form_validation_rejected_count"] == 1
|
||||||
|
|
||||||
|
|
||||||
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
|
def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
|
||||||
@@ -836,6 +864,57 @@ def test_grammar_validation_accepts_meaning_preserving_fix(tmp_path):
|
|||||||
|
|
||||||
assert revised[0].text == "Keep in mind."
|
assert revised[0].text == "Keep in mind."
|
||||||
assert fake_client.validation_calls == 1
|
assert fake_client.validation_calls == 1
|
||||||
|
assert fake_client.spoken_form_validation_calls == 0
|
||||||
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_validation_rescues_likely_spoken_form_fix(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
grammar_correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="dam",
|
||||||
|
corrected_text="damn",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
meaning_validation = GrammarValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_meaning_preserving=False,
|
||||||
|
confidence=0.98,
|
||||||
|
reason="Written meaning changes from a barrier to a curse word.",
|
||||||
|
)
|
||||||
|
spoken_form_validation = GrammarSpokenFormValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_likely_spoken_form_correction=True,
|
||||||
|
confidence=0.97,
|
||||||
|
reason="The phrase strongly suggests the intended spoken word was the expletive.",
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
CorrectionSet(corrections=[grammar_correction]),
|
||||||
|
],
|
||||||
|
validation_responses=[GrammarValidationSet(validations=[meaning_validation])],
|
||||||
|
spoken_form_validation_responses=[
|
||||||
|
GrammarSpokenFormValidationSet(validations=[spoken_form_validation])
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
revised = process_transcript(
|
||||||
|
transcript,
|
||||||
|
_glossary(),
|
||||||
|
_config(tmp_path, grammar_validation_enabled=True),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert revised[0].text == "ChatGPT still can't really do that with a damn."
|
||||||
|
assert fake_client.validation_calls == 1
|
||||||
|
assert fake_client.spoken_form_validation_calls == 1
|
||||||
assert list((tmp_path / "work").iterdir()) == []
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
@@ -877,6 +956,7 @@ def test_grammar_validation_bypasses_protected_vocabulary_correction(tmp_path):
|
|||||||
|
|
||||||
assert revised[0].text == "The Jesters arrived."
|
assert revised[0].text == "The Jesters arrived."
|
||||||
assert fake_client.validation_calls == 0
|
assert fake_client.validation_calls == 0
|
||||||
|
assert fake_client.spoken_form_validation_calls == 0
|
||||||
assert list((tmp_path / "work").iterdir()) == []
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
@@ -910,6 +990,7 @@ def test_disabled_grammar_validation_preserves_current_behavior(tmp_path):
|
|||||||
|
|
||||||
assert revised[0].text == "He became invisible."
|
assert revised[0].text == "He became invisible."
|
||||||
assert fake_client.validation_calls == 0
|
assert fake_client.validation_calls == 0
|
||||||
|
assert fake_client.spoken_form_validation_calls == 0
|
||||||
assert list((tmp_path / "work").iterdir()) == []
|
assert list((tmp_path / "work").iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
@@ -949,6 +1030,54 @@ def test_missing_grammar_validation_decision_fails_and_preserves_diagnostics(tmp
|
|||||||
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
|
assert (run_dirs[0] / "grammar" / "pass-0001" / "validation-response-0000.json").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_spoken_form_validation_decision_fails_and_preserves_diagnostics(tmp_path):
|
||||||
|
transcript = parse_source_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
grammar_correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="dam",
|
||||||
|
corrected_text="damn",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
fake_client = FakeLLMClient(
|
||||||
|
[
|
||||||
|
CorrectionSet(corrections=[]),
|
||||||
|
CorrectionSet(corrections=[grammar_correction]),
|
||||||
|
],
|
||||||
|
validation_responses=[
|
||||||
|
GrammarValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_meaning_preserving=False,
|
||||||
|
confidence=0.99,
|
||||||
|
reason="Written meaning changes.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
],
|
||||||
|
spoken_form_validation_responses=[GrammarSpokenFormValidationSet(validations=[])],
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AuditaError):
|
||||||
|
process_transcript(
|
||||||
|
transcript,
|
||||||
|
_glossary(),
|
||||||
|
_config(tmp_path, grammar_validation_enabled=True),
|
||||||
|
llm_client=fake_client,
|
||||||
|
)
|
||||||
|
|
||||||
|
run_dirs = list((tmp_path / "work").iterdir())
|
||||||
|
assert len(run_dirs) == 1
|
||||||
|
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-prompt-0000.json").exists()
|
||||||
|
assert (run_dirs[0] / "grammar" / "pass-0001" / "spoken-form-validation-response-0000.json").exists()
|
||||||
|
|
||||||
|
|
||||||
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
|
def test_grammar_stage_cannot_reverse_glossary_protected_term(tmp_path):
|
||||||
transcript = parse_source_transcript_json(
|
transcript = parse_source_transcript_json(
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from audita.chunking import chunk_transcript
|
|||||||
from audita.prompts import (
|
from audita.prompts import (
|
||||||
build_glossary_correction_messages,
|
build_glossary_correction_messages,
|
||||||
build_grammar_correction_messages,
|
build_grammar_correction_messages,
|
||||||
|
build_grammar_spoken_form_validation_messages,
|
||||||
build_grammar_validation_messages,
|
build_grammar_validation_messages,
|
||||||
)
|
)
|
||||||
from audita.schemas import parse_glossary_yaml, parse_transcript_json
|
from audita.schemas import parse_glossary_yaml, parse_transcript_json
|
||||||
@@ -194,6 +195,28 @@ def test_grammar_validation_prompt_rejects_semantic_changes():
|
|||||||
assert "became visible" in prompt_text
|
assert "became visible" in prompt_text
|
||||||
assert "became invisible" in prompt_text
|
assert "became invisible" in prompt_text
|
||||||
assert "reverses the meaning" in prompt_text
|
assert "reverses the meaning" in prompt_text
|
||||||
assert "spelling and homophone fixes only when" in prompt_text
|
assert "do not try to rescue likely homophone or transcription fixes" in prompt_text
|
||||||
|
assert "handled in a separate spoken-form validation step" in prompt_text
|
||||||
assert "correction_index" in prompt_text
|
assert "correction_index" in prompt_text
|
||||||
assert "is_meaning_preserving" in prompt_text
|
assert "is_meaning_preserving" in prompt_text
|
||||||
|
|
||||||
|
|
||||||
|
def test_grammar_spoken_form_validation_prompt_allows_homophone_rescue():
|
||||||
|
messages = build_grammar_spoken_form_validation_messages(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"correction_index": 0,
|
||||||
|
"id": 1,
|
||||||
|
"original_segment_text": "ChatGPT still can't really do that with a dam.",
|
||||||
|
"corrected_segment_text": "ChatGPT still can't really do that with a damn.",
|
||||||
|
"original_text": "dam",
|
||||||
|
"corrected_text": "damn",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
prompt_text = "\n".join(message["content"] for message in messages)
|
||||||
|
|
||||||
|
assert "likely homophone, spoken-form, or transcription fix" in prompt_text
|
||||||
|
assert '"dam" to "damn"' in prompt_text
|
||||||
|
assert '"became visible" to "became invisible"' in prompt_text
|
||||||
|
assert "is_likely_spoken_form_correction" in prompt_text
|
||||||
|
|||||||
@@ -1,16 +1,22 @@
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from audita.errors import AuditaLLMError
|
from audita.errors import AuditaLLMError
|
||||||
from audita.semantic_validation import select_grammar_validation_candidates
|
|
||||||
from audita.semantic_validation import filter_with_grammar_validations
|
|
||||||
from audita.protection import ProtectedVocabulary
|
from audita.protection import ProtectedVocabulary
|
||||||
from audita.schemas import (
|
from audita.schemas import (
|
||||||
CorrectionCandidate,
|
CorrectionCandidate,
|
||||||
|
GrammarSpokenFormValidationDecision,
|
||||||
|
GrammarSpokenFormValidationSet,
|
||||||
GrammarValidationDecision,
|
GrammarValidationDecision,
|
||||||
GrammarValidationSet,
|
GrammarValidationSet,
|
||||||
parse_glossary_yaml,
|
parse_glossary_yaml,
|
||||||
parse_transcript_json,
|
parse_transcript_json,
|
||||||
)
|
)
|
||||||
|
from audita.semantic_validation import (
|
||||||
|
filter_with_meaning_preserving_validations,
|
||||||
|
filter_with_spoken_form_validations,
|
||||||
|
keep_corrections_with_indexes,
|
||||||
|
select_grammar_validation_candidates,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _transcript():
|
def _transcript():
|
||||||
@@ -118,7 +124,106 @@ def test_meaning_sensitive_substitution_requires_validation():
|
|||||||
assert bypassed_count == 0
|
assert bypassed_count == 0
|
||||||
|
|
||||||
|
|
||||||
def test_validation_rejects_duplicate_and_unknown_decisions():
|
def test_meaning_preserving_validation_approves_without_rescue():
|
||||||
|
correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="bind",
|
||||||
|
corrected_text="mind",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
candidates, _ = select_grammar_validation_candidates(
|
||||||
|
transcript,
|
||||||
|
[correction],
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
replacement_mode="require_unique",
|
||||||
|
protected_vocabulary=_vocabulary(),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = filter_with_meaning_preserving_validations(
|
||||||
|
candidates,
|
||||||
|
GrammarValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_meaning_preserving=True,
|
||||||
|
confidence=0.95,
|
||||||
|
reason="This preserves the intended meaning.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.approved_correction_indexes == [0]
|
||||||
|
assert result.rescue_candidates == []
|
||||||
|
assert result.approved_count == 1
|
||||||
|
assert result.rejected_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_spoken_form_validation_can_rescue_homophone_fix():
|
||||||
|
correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="dam",
|
||||||
|
corrected_text="damn",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
transcript = parse_transcript_json(
|
||||||
|
"""
|
||||||
|
[
|
||||||
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
|
||||||
|
]
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
candidates, _ = select_grammar_validation_candidates(
|
||||||
|
transcript,
|
||||||
|
[correction],
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
replacement_mode="require_unique",
|
||||||
|
protected_vocabulary=_vocabulary(),
|
||||||
|
)
|
||||||
|
meaning_result = filter_with_meaning_preserving_validations(
|
||||||
|
candidates,
|
||||||
|
GrammarValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_meaning_preserving=False,
|
||||||
|
confidence=0.99,
|
||||||
|
reason="Written meaning changes from a barrier to a curse word.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
spoken_form_result = filter_with_spoken_form_validations(
|
||||||
|
meaning_result.rescue_candidates,
|
||||||
|
GrammarSpokenFormValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarSpokenFormValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_likely_spoken_form_correction=True,
|
||||||
|
confidence=0.95,
|
||||||
|
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
|
||||||
|
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
|
||||||
|
assert meaning_result.approved_correction_indexes == []
|
||||||
|
assert kept == [correction]
|
||||||
|
assert spoken_form_result.skipped == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
|
||||||
correction = CorrectionCandidate(
|
correction = CorrectionCandidate(
|
||||||
id=1,
|
id=1,
|
||||||
original_text="visible",
|
original_text="visible",
|
||||||
@@ -132,30 +237,90 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
|
|||||||
replacement_mode="require_unique",
|
replacement_mode="require_unique",
|
||||||
protected_vocabulary=_vocabulary(),
|
protected_vocabulary=_vocabulary(),
|
||||||
)
|
)
|
||||||
decision = GrammarValidationDecision(
|
meaning_result = filter_with_meaning_preserving_validations(
|
||||||
|
candidates,
|
||||||
|
GrammarValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_meaning_preserving=False,
|
||||||
|
confidence=0.99,
|
||||||
|
reason="This reverses visible to invisible.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
spoken_form_result = filter_with_spoken_form_validations(
|
||||||
|
meaning_result.rescue_candidates,
|
||||||
|
GrammarSpokenFormValidationSet(
|
||||||
|
validations=[
|
||||||
|
GrammarSpokenFormValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_likely_spoken_form_correction=False,
|
||||||
|
confidence=0.99,
|
||||||
|
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert spoken_form_result.approved_correction_indexes == []
|
||||||
|
assert spoken_form_result.rejected_count == 1
|
||||||
|
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
|
||||||
|
assert spoken_form_result.skipped[0].validation_reason == (
|
||||||
|
"This is a semantic reversal, not a likely spoken-form transcription error."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
|
||||||
|
correction = CorrectionCandidate(
|
||||||
|
id=1,
|
||||||
|
original_text="visible",
|
||||||
|
corrected_text="invisible",
|
||||||
|
confidence=0.95,
|
||||||
|
)
|
||||||
|
candidates, _ = select_grammar_validation_candidates(
|
||||||
|
_transcript(),
|
||||||
|
[correction],
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
replacement_mode="require_unique",
|
||||||
|
protected_vocabulary=_vocabulary(),
|
||||||
|
)
|
||||||
|
meaning_decision = GrammarValidationDecision(
|
||||||
correction_index=0,
|
correction_index=0,
|
||||||
is_meaning_preserving=True,
|
is_meaning_preserving=True,
|
||||||
confidence=0.95,
|
confidence=0.95,
|
||||||
reason="Preserves meaning.",
|
reason="Preserves meaning.",
|
||||||
)
|
)
|
||||||
|
spoken_form_decision = GrammarSpokenFormValidationDecision(
|
||||||
|
correction_index=0,
|
||||||
|
is_likely_spoken_form_correction=True,
|
||||||
|
confidence=0.95,
|
||||||
|
reason="Likely spoken-form correction.",
|
||||||
|
)
|
||||||
|
|
||||||
with pytest.raises(AuditaLLMError):
|
with pytest.raises(AuditaLLMError):
|
||||||
filter_with_grammar_validations(
|
filter_with_meaning_preserving_validations(
|
||||||
[correction],
|
|
||||||
candidates,
|
candidates,
|
||||||
GrammarValidationSet(validations=[decision, decision]),
|
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
|
||||||
confidence_threshold=0.8,
|
confidence_threshold=0.8,
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(AuditaLLMError):
|
with pytest.raises(AuditaLLMError):
|
||||||
filter_with_grammar_validations(
|
filter_with_meaning_preserving_validations(
|
||||||
[correction],
|
|
||||||
candidates,
|
candidates,
|
||||||
GrammarValidationSet(
|
GrammarValidationSet(validations=[]),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
with pytest.raises(AuditaLLMError):
|
||||||
|
filter_with_spoken_form_validations(
|
||||||
|
candidates,
|
||||||
|
GrammarSpokenFormValidationSet(
|
||||||
validations=[
|
validations=[
|
||||||
GrammarValidationDecision(
|
GrammarSpokenFormValidationDecision(
|
||||||
correction_index=99,
|
correction_index=99,
|
||||||
is_meaning_preserving=True,
|
is_likely_spoken_form_correction=True,
|
||||||
confidence=0.95,
|
confidence=0.95,
|
||||||
reason="Unknown.",
|
reason="Unknown.",
|
||||||
)
|
)
|
||||||
@@ -163,3 +328,9 @@ def test_validation_rejects_duplicate_and_unknown_decisions():
|
|||||||
),
|
),
|
||||||
confidence_threshold=0.8,
|
confidence_threshold=0.8,
|
||||||
)
|
)
|
||||||
|
with pytest.raises(AuditaLLMError):
|
||||||
|
filter_with_spoken_form_validations(
|
||||||
|
candidates,
|
||||||
|
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
|
||||||
|
confidence_threshold=0.8,
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user