Updated configuration to utilize segment ids provided in the input transcript

This commit is contained in:
2026-04-21 15:01:31 -05:00
parent 7e1a3f721a
commit ca01e46d77
13 changed files with 186 additions and 97 deletions

View File

@@ -42,7 +42,7 @@ Useful configuration can be supplied by CLI flag or environment variable:
- `AUDITA_MODEL`, default `openrouter/mistralai/mistral-small-3.2-24b-instruct` - `AUDITA_MODEL`, default `openrouter/mistralai/mistral-small-3.2-24b-instruct`
- `AUDITA_BASE_URL`, default `https://openrouter.ai/api/v1` - `AUDITA_BASE_URL`, default `https://openrouter.ai/api/v1`
- `AUDITA_MAX_SECTION_TOKENS`, default `16000` - `AUDITA_MAX_SECTION_TOKENS`, default `16000`
- `AUDITA_CONFIDENCE_THRESHOLD`, default `0.80` - `AUDITA_CONFIDENCE_THRESHOLD`, default `0.60`
- `AUDITA_MAX_RETRIES`, default `3` - `AUDITA_MAX_RETRIES`, default `3`
- `AUDITA_GLOSSARY_MAX_LLM_PASSES`, default `3`, for total glossary correction passes - `AUDITA_GLOSSARY_MAX_LLM_PASSES`, default `3`, for total glossary correction passes
- `AUDITA_WORK_DIR`, default `/tmp/audita` - `AUDITA_WORK_DIR`, default `/tmp/audita`

View File

@@ -41,7 +41,7 @@ class IndexedSegment:
return self.segment.model_dump(mode="json") return self.segment.model_dump(mode="json")
def prompt_payload(self) -> dict: def prompt_payload(self) -> dict:
return {"segment_id": self.index, "original_text": self.segment.text} return {"id": self.segment.id, "original_text": self.segment.text}
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -116,7 +116,7 @@ def chunk_indexed_segments(
sections.append(_make_section(len(sections), current, current_tokens)) sections.append(_make_section(len(sections), current, current_tokens))
for section in sections: for section in sections:
parse_transcript_json(section.transcript_json()) parse_transcript_json(section.transcript_json(), require_sequential_ids=False)
return sections return sections

View File

@@ -9,7 +9,7 @@ from .errors import AuditaConfigError
DEFAULT_MODEL = "openrouter/mistralai/mistral-small-3.2-24b-instruct" DEFAULT_MODEL = "openrouter/mistralai/mistral-small-3.2-24b-instruct"
DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" DEFAULT_BASE_URL = "https://openrouter.ai/api/v1"
DEFAULT_MAX_SECTION_TOKENS = 16000 DEFAULT_MAX_SECTION_TOKENS = 16000
DEFAULT_CONFIDENCE_THRESHOLD = 0.80 DEFAULT_CONFIDENCE_THRESHOLD = 0.60
DEFAULT_MAX_RETRIES = 3 DEFAULT_MAX_RETRIES = 3
DEFAULT_WORK_DIR = "/tmp/audita" DEFAULT_WORK_DIR = "/tmp/audita"
DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3 DEFAULT_GLOSSARY_MAX_LLM_PASSES = 3

View File

@@ -1,5 +1,5 @@
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from typing import Iterable, List, Optional, Tuple from typing import Dict, Iterable, List, Optional, Tuple
from .errors import AuditaValidationError from .errors import AuditaValidationError
from .schemas import CorrectionCandidate, TranscriptSegment from .schemas import CorrectionCandidate, TranscriptSegment
@@ -7,7 +7,7 @@ from .schemas import CorrectionCandidate, TranscriptSegment
@dataclass(frozen=True) @dataclass(frozen=True)
class SkippedCorrection: class SkippedCorrection:
segment_id: int id: int
reason: str reason: str
original_text: str original_text: str
corrected_text: str corrected_text: str
@@ -22,8 +22,8 @@ class SkippedCorrection:
class CorrectionApplicationResult: class CorrectionApplicationResult:
transcript: List[TranscriptSegment] transcript: List[TranscriptSegment]
skipped: List[SkippedCorrection] skipped: List[SkippedCorrection]
applied_segment_ids: List[int] applied_ids: List[int]
ignored_segment_ids: List[int] ignored_ids: List[int]
def apply_corrections( def apply_corrections(
@@ -35,40 +35,43 @@ def apply_corrections(
raise AuditaValidationError("Confidence threshold must be between 0.0 and 1.0.") raise AuditaValidationError("Confidence threshold must be between 0.0 and 1.0.")
revised = list(transcript) revised = list(transcript)
id_to_position = _id_to_position(revised)
skipped: List[SkippedCorrection] = [] skipped: List[SkippedCorrection] = []
applied_segment_ids: List[int] = [] applied_ids: List[int] = []
ignored_segment_ids: List[int] = [] ignored_ids: List[int] = []
for correction in corrections: for correction in corrections:
if correction.confidence < confidence_threshold: if correction.confidence < confidence_threshold:
ignored_segment_ids.append(correction.segment_id) ignored_ids.append(correction.id)
continue continue
reason, actual_text = _target_error(revised, correction) reason, actual_text = _target_error(revised, id_to_position, correction)
if reason is not None: if reason is not None:
skipped.append(_skip(correction, reason, actual_text=actual_text)) skipped.append(_skip(correction, reason, actual_text=actual_text))
continue continue
segment = revised[correction.segment_id] position = id_to_position[correction.id]
revised_text = segment.text.replace(correction.original_text, correction.corrected_text, 1) segment = revised[position]
revised[correction.segment_id] = segment.model_copy(update={"text": revised_text}) revised_text = segment.text.replace(correction.original_text, correction.corrected_text)
applied_segment_ids.append(correction.segment_id) revised[position] = segment.model_copy(update={"text": revised_text})
applied_ids.append(correction.id)
return CorrectionApplicationResult( return CorrectionApplicationResult(
transcript=revised, transcript=revised,
skipped=skipped, skipped=skipped,
applied_segment_ids=applied_segment_ids, applied_ids=applied_ids,
ignored_segment_ids=ignored_segment_ids, ignored_ids=ignored_ids,
) )
def _target_error( def _target_error(
transcript: List[TranscriptSegment], transcript: List[TranscriptSegment],
id_to_position: Dict[int, int],
correction: CorrectionCandidate, correction: CorrectionCandidate,
) -> Tuple[Optional[str], Optional[str]]: ) -> Tuple[Optional[str], Optional[str]]:
if correction.segment_id >= len(transcript): if correction.id not in id_to_position:
return "segment_id does not exist in transcript", None return "id does not exist in transcript", None
segment = transcript[correction.segment_id] segment = transcript[id_to_position[correction.id]]
if correction.original_text == "": if correction.original_text == "":
return "original_text is empty", segment.text return "original_text is empty", segment.text
if correction.original_text == correction.corrected_text: if correction.original_text == correction.corrected_text:
@@ -77,19 +80,21 @@ def _target_error(
match_count = segment.text.count(correction.original_text) match_count = segment.text.count(correction.original_text)
if match_count == 0: if match_count == 0:
return "original_text does not match any substring in segment text", segment.text return "original_text does not match any substring in segment text", segment.text
if match_count > 1:
return "original_text appears multiple times in segment text", segment.text
return None, None return None, None
def _id_to_position(transcript: List[TranscriptSegment]) -> Dict[int, int]:
return {segment.id: position for position, segment in enumerate(transcript)}
def _skip( def _skip(
correction: CorrectionCandidate, correction: CorrectionCandidate,
reason: str, reason: str,
actual_text: Optional[str] = None, actual_text: Optional[str] = None,
) -> SkippedCorrection: ) -> SkippedCorrection:
return SkippedCorrection( return SkippedCorrection(
segment_id=correction.segment_id, id=correction.id,
reason=reason, reason=reason,
original_text=correction.original_text, original_text=correction.original_text,
corrected_text=correction.corrected_text, corrected_text=correction.corrected_text,

View File

@@ -41,12 +41,12 @@ def process_transcript(
for pass_number in range(1, config.glossary_max_llm_passes + 1): for pass_number in range(1, config.glossary_max_llm_passes + 1):
if pass_number == 1: if pass_number == 1:
indexed_segments = _indexed_segments_for_ids(working, list(range(len(working)))) indexed_segments = _indexed_segments_for_ids(working, [segment.id for segment in working])
else: else:
retry_segment_ids = sorted(unresolved_retry_skips) retry_ids = sorted(unresolved_retry_skips)
if not retry_segment_ids: if not retry_ids:
break break
indexed_segments = _indexed_segments_for_ids(working, retry_segment_ids) indexed_segments = _indexed_segments_for_ids(working, retry_ids)
if not indexed_segments: if not indexed_segments:
break break
@@ -80,13 +80,13 @@ def process_transcript(
) )
working = application_result.transcript working = application_result.transcript
for segment_id in application_result.applied_segment_ids: for correction_id in application_result.applied_ids:
unresolved_retry_skips.pop(segment_id, None) unresolved_retry_skips.pop(correction_id, None)
for segment_id in application_result.ignored_segment_ids: for correction_id in application_result.ignored_ids:
unresolved_retry_skips.pop(segment_id, None) unresolved_retry_skips.pop(correction_id, None)
for skipped in application_result.skipped: for skipped in application_result.skipped:
if _is_retryable_skip(skipped, len(working)): if _is_retryable_skip(skipped, working):
unresolved_retry_skips[skipped.segment_id] = skipped unresolved_retry_skips[skipped.id] = skipped
else: else:
final_nonretry_skips.append(skipped) final_nonretry_skips.append(skipped)
@@ -97,8 +97,8 @@ def process_transcript(
"section_count": len(sections), "section_count": len(sections),
"segment_count": len(indexed_segments), "segment_count": len(indexed_segments),
"corrections_returned": len(corrections), "corrections_returned": len(corrections),
"applied_count": len(application_result.applied_segment_ids), "applied_count": len(application_result.applied_ids),
"ignored_below_threshold_count": len(application_result.ignored_segment_ids), "ignored_below_threshold_count": len(application_result.ignored_ids),
"skipped_count": len(application_result.skipped), "skipped_count": len(application_result.skipped),
"retry_segment_count": len(unresolved_retry_skips), "retry_segment_count": len(unresolved_retry_skips),
} }
@@ -109,13 +109,13 @@ def process_transcript(
break break
final_skipped = final_nonretry_skips + [ final_skipped = final_nonretry_skips + [
unresolved_retry_skips[segment_id] for segment_id in sorted(unresolved_retry_skips) unresolved_retry_skips[correction_id] for correction_id in sorted(unresolved_retry_skips)
] ]
_write_skipped_corrections(run_dir, final_skipped) _write_skipped_corrections(run_dir, final_skipped)
for skipped in final_skipped: for skipped in final_skipped:
_log( _log(
progress, progress,
f"Skipping correction for segment {skipped.segment_id}: {skipped.reason}", f"Skipping correction for id {skipped.id}: {skipped.reason}",
) )
revised = _sort_transcript_chronologically(working) revised = _sort_transcript_chronologically(working)
except Exception as exc: except Exception as exc:
@@ -164,7 +164,7 @@ def _write_and_validate_section(run_dir: Path, section: TranscriptSection) -> No
section_path = run_dir / f"section-{section.section_index:04d}.json" section_path = run_dir / f"section-{section.section_index:04d}.json"
section_json = section.transcript_json() section_json = section.transcript_json()
section_path.write_text(section_json, encoding="utf-8") section_path.write_text(section_json, encoding="utf-8")
parse_transcript_json(section_json) parse_transcript_json(section_json, require_sequential_ids=False)
def _write_skipped_corrections(run_dir: Path, skipped: List[SkippedCorrection]) -> None: def _write_skipped_corrections(run_dir: Path, skipped: List[SkippedCorrection]) -> None:
@@ -182,17 +182,18 @@ def _write_skipped_corrections(run_dir: Path, skipped: List[SkippedCorrection])
def _indexed_segments_for_ids( def _indexed_segments_for_ids(
transcript: List[TranscriptSegment], transcript: List[TranscriptSegment],
segment_ids: List[int], ids: List[int],
) -> List[IndexedSegment]: ) -> List[IndexedSegment]:
id_to_position = {segment.id: position for position, segment in enumerate(transcript)}
return [ return [
IndexedSegment(index=segment_id, segment=transcript[segment_id]) IndexedSegment(index=id_to_position[segment_id], segment=transcript[id_to_position[segment_id]])
for segment_id in segment_ids for segment_id in ids
if 0 <= segment_id < len(transcript) if segment_id in id_to_position
] ]
def _is_retryable_skip(skipped: SkippedCorrection, transcript_length: int) -> bool: def _is_retryable_skip(skipped: SkippedCorrection, transcript: List[TranscriptSegment]) -> bool:
return 0 <= skipped.segment_id < transcript_length return any(segment.id == skipped.id for segment in transcript)
def _sort_transcript_chronologically( def _sort_transcript_chronologically(

View File

@@ -44,10 +44,10 @@ def build_glossary_correction_messages(
"- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n" "- Inappropriate example: correcting \"Lyra\" to \"Jesters\" should be omitted because those words are not similar in spoken English, even if \"Jesters\" appears in the glossary.\n"
"- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n" "- Do not replace one clear glossary term, character name, location, or ordinary word with a different glossary term unless it is a plausible mishearing.\n"
"- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n" "- Assign high confidence only when the correction is supported by glossary evidence, local context, and spoken-word similarity; otherwise omit the correction.\n"
"- Use the exact segment_id from the input segment.\n" "- Use the exact id from the input segment.\n"
"- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n" "- For returned corrections, original_text must be only the exact text span that needs replacement, not the full segment text unless the whole segment is the replacement span.\n"
"- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n" "- corrected_text must be only the replacement text for that span, not the full corrected segment text unless the whole segment is the replacement span.\n"
"- Each returned correction must contain only segment_id, original_text, corrected_text, and confidence.\n" "- Each returned correction must contain only id, original_text, corrected_text, and confidence.\n"
"- Do not return corrections where original_text and corrected_text are identical.\n" "- Do not return corrections where original_text and corrected_text are identical.\n"
"- Do not return speaker, start, or end fields.\n" "- Do not return speaker, start, or end fields.\n"
"- Return only changed segments; do not return entries for unchanged segments.\n" "- Return only changed segments; do not return entries for unchanged segments.\n"

View File

@@ -11,11 +11,19 @@ from .errors import AuditaValidationError
class TranscriptSegment(BaseModel): class TranscriptSegment(BaseModel):
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
id: int = Field(ge=1)
speaker: StrictStr speaker: StrictStr
start: float start: float
end: float end: float
text: StrictStr text: StrictStr
@field_validator("id", mode="before")
@classmethod
def require_integer_id(cls, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError("must be an integer")
return value
@field_validator("speaker", "text") @field_validator("speaker", "text")
@classmethod @classmethod
def require_non_empty_text(cls, value: str) -> str: def require_non_empty_text(cls, value: str) -> str:
@@ -82,12 +90,12 @@ class Glossary(BaseModel):
class CorrectionCandidate(BaseModel): class CorrectionCandidate(BaseModel):
model_config = ConfigDict(extra="forbid") model_config = ConfigDict(extra="forbid")
segment_id: int = Field(ge=0) id: int = Field(ge=1)
original_text: StrictStr original_text: StrictStr
corrected_text: StrictStr corrected_text: StrictStr
confidence: float = Field(ge=0.0, le=1.0) confidence: float = Field(ge=0.0, le=1.0)
@field_validator("segment_id", mode="before") @field_validator("id", mode="before")
@classmethod @classmethod
def require_integer_id(cls, value: Any) -> int: def require_integer_id(cls, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int): if isinstance(value, bool) or not isinstance(value, int):
@@ -114,23 +122,39 @@ class CorrectionSet(BaseModel):
_TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment]) _TRANSCRIPT_ADAPTER = TypeAdapter(List[TranscriptSegment])
def validate_transcript_data(data: Any) -> List[TranscriptSegment]: def validate_transcript_data(
data: Any,
require_sequential_ids: bool = True,
) -> List[TranscriptSegment]:
if not isinstance(data, list): if not isinstance(data, list):
raise AuditaValidationError("Transcript must be a JSON array.") raise AuditaValidationError("Transcript must be a JSON array.")
if not data: if not data:
raise AuditaValidationError("Transcript must contain at least one segment.") raise AuditaValidationError("Transcript must contain at least one segment.")
try: try:
return _TRANSCRIPT_ADAPTER.validate_python(data) transcript = _TRANSCRIPT_ADAPTER.validate_python(data)
except ValidationError as exc: except ValidationError as exc:
raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc raise AuditaValidationError(f"Transcript schema validation failed: {exc}") from exc
if require_sequential_ids:
_validate_sequential_ids(transcript)
return transcript
def parse_transcript_json(raw: str) -> List[TranscriptSegment]: def _validate_sequential_ids(transcript: List[TranscriptSegment]) -> None:
ids = [segment.id for segment in transcript]
expected = list(range(1, len(transcript) + 1))
if ids != expected:
raise AuditaValidationError("Transcript segment ids must be sequential starting at 1.")
def parse_transcript_json(
raw: str,
require_sequential_ids: bool = True,
) -> List[TranscriptSegment]:
try: try:
data = json.loads(raw) data = json.loads(raw)
except json.JSONDecodeError as exc: except json.JSONDecodeError as exc:
raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc raise AuditaValidationError(f"Transcript is not valid JSON: {exc}") from exc
return validate_transcript_data(data) return validate_transcript_data(data, require_sequential_ids=require_sequential_ids)
def parse_glossary_yaml(raw: str) -> Glossary: def parse_glossary_yaml(raw: str) -> Glossary:

View File

@@ -12,7 +12,7 @@ class CountEstimator:
def _segments(count): def _segments(count):
payload = [ payload = [
{"speaker": "Eric", "start": float(i), "end": float(i + 1), "text": f"Segment {i}"} {"id": i + 1, "speaker": "Eric", "start": float(i), "end": float(i + 1), "text": f"Segment {i}"}
for i in range(count) for i in range(count)
] ]
import json import json
@@ -37,4 +37,3 @@ def test_chunk_transcript_allows_exact_limit():
def test_chunk_transcript_rejects_oversized_single_segment(): def test_chunk_transcript_rejects_oversized_single_segment():
with pytest.raises(AuditaValidationError): with pytest.raises(AuditaValidationError):
chunk_transcript(_segments(1), max_section_tokens=9, estimator=CountEstimator()) chunk_transcript(_segments(1), max_section_tokens=9, estimator=CountEstimator())

View File

@@ -3,13 +3,15 @@ from pathlib import Path
import pytest import pytest
from audita.config import AuditaConfig, ConfigOverrides from audita.config import AuditaConfig, ConfigOverrides
from audita.config import DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR from audita.config import DEFAULT_CONFIDENCE_THRESHOLD, DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR
from audita.errors import AuditaConfigError from audita.errors import AuditaConfigError
def test_config_uses_defaults_with_api_key(): def test_config_uses_defaults_with_api_key():
config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"}) config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
assert config.confidence_threshold == DEFAULT_CONFIDENCE_THRESHOLD
assert config.confidence_threshold == 0.6
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.max_retries == DEFAULT_MAX_RETRIES assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
@@ -46,6 +48,7 @@ def test_config_cli_overrides_env():
}, },
overrides=ConfigOverrides( overrides=ConfigOverrides(
max_section_tokens=100, max_section_tokens=100,
confidence_threshold=0.7,
max_retries=3, max_retries=3,
glossary_max_llm_passes=2, glossary_max_llm_passes=2,
work_dir=Path("/tmp/cli-audita"), work_dir=Path("/tmp/cli-audita"),
@@ -53,6 +56,7 @@ def test_config_cli_overrides_env():
) )
assert config.max_section_tokens == 100 assert config.max_section_tokens == 100
assert config.confidence_threshold == 0.7
assert config.max_retries == 3 assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2 assert config.glossary_max_llm_passes == 2
assert config.work_dir == Path("/tmp/cli-audita") assert config.work_dir == Path("/tmp/cli-audita")

View File

@@ -9,18 +9,18 @@ def _transcript():
return parse_transcript_json( return parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia for help."}, {"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia for help."},
{"speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."} {"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
] ]
""" """
) )
def test_apply_corrections_uses_threshold_and_preserves_segment_id_order(): def test_apply_corrections_uses_threshold_and_preserves_id_order():
transcript = _transcript() transcript = _transcript()
corrections = [ corrections = [
CorrectionCandidate( CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.8, confidence=0.8,
@@ -38,7 +38,7 @@ def test_apply_corrections_ignores_below_threshold():
transcript = _transcript() transcript = _transcript()
corrections = [ corrections = [
CorrectionCandidate( CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.79, confidence=0.79,
@@ -54,13 +54,13 @@ def test_apply_corrections_ignores_below_threshold():
def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment(): def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
transcript = _transcript() transcript = _transcript()
first = CorrectionCandidate( first = CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.8, confidence=0.8,
) )
second = CorrectionCandidate( second = CorrectionCandidate(
segment_id=0, id=1,
original_text="help", original_text="help",
corrected_text="guidance", corrected_text="guidance",
confidence=0.9, confidence=0.9,
@@ -75,7 +75,7 @@ def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
def test_apply_corrections_skips_missing_substring(): def test_apply_corrections_skips_missing_substring():
transcript = _transcript() transcript = _transcript()
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="Different text.", original_text="Different text.",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.8, confidence=0.8,
@@ -85,15 +85,15 @@ def test_apply_corrections_skips_missing_substring():
assert result.transcript[0].text == "I ask Chontia for help." assert result.transcript[0].text == "I ask Chontia for help."
assert len(result.skipped) == 1 assert len(result.skipped) == 1
assert result.skipped[0].segment_id == 0 assert result.skipped[0].id == 1
assert result.skipped[0].actual_text == "I ask Chontia for help." assert result.skipped[0].actual_text == "I ask Chontia for help."
assert "does not match any substring" in result.skipped[0].reason assert "does not match any substring" in result.skipped[0].reason
def test_apply_corrections_skips_missing_segment_id(): def test_apply_corrections_skips_missing_id():
transcript = _transcript() transcript = _transcript()
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=99, id=99,
original_text="Missing.", original_text="Missing.",
corrected_text="Still missing.", corrected_text="Still missing.",
confidence=0.8, confidence=0.8,
@@ -103,14 +103,14 @@ def test_apply_corrections_skips_missing_segment_id():
assert [segment.text for segment in result.transcript] == ["I ask Chontia for help.", "Then Lyra."] assert [segment.text for segment in result.transcript] == ["I ask Chontia for help.", "Then Lyra."]
assert len(result.skipped) == 1 assert len(result.skipped) == 1
assert result.skipped[0].segment_id == 99 assert result.skipped[0].id == 99
assert "does not exist" in result.skipped[0].reason assert "does not exist" in result.skipped[0].reason
def test_apply_corrections_skips_no_op(): def test_apply_corrections_skips_no_op():
transcript = _transcript() transcript = _transcript()
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chontia", corrected_text="Chontia",
confidence=0.8, confidence=0.8,
@@ -123,16 +123,16 @@ def test_apply_corrections_skips_no_op():
assert "identical" in result.skipped[0].reason assert "identical" in result.skipped[0].reason
def test_apply_corrections_skips_ambiguous_repeated_substring(): def test_apply_corrections_replaces_all_repeated_substrings():
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Bane met Bane."} {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Bane met Bane."}
] ]
""" """
) )
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="Bane", original_text="Bane",
corrected_text="Bain", corrected_text="Bain",
confidence=0.8, confidence=0.8,
@@ -140,15 +140,14 @@ def test_apply_corrections_skips_ambiguous_repeated_substring():
result = apply_corrections(transcript, [correction], confidence_threshold=0.8) result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "Bane met Bane." assert result.transcript[0].text == "Bain met Bain."
assert len(result.skipped) == 1 assert result.skipped == []
assert "multiple times" in result.skipped[0].reason
def test_apply_corrections_skips_empty_original_text(): def test_apply_corrections_skips_empty_original_text():
transcript = _transcript() transcript = _transcript()
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="", original_text="",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.8, confidence=0.8,

View File

@@ -43,8 +43,8 @@ def _transcript():
return parse_transcript_json( return parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."}, {"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
{"speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."} {"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
] ]
""" """
) )
@@ -52,7 +52,7 @@ def _transcript():
def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path): def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
@@ -74,7 +74,7 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path): def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate( correction = CorrectionCandidate(
segment_id=0, id=1,
original_text="Different text.", original_text="Different text.",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
@@ -91,25 +91,25 @@ def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
) )
assert revised[1].text == "I ask Chontia." assert revised[1].text == "I ask Chontia."
assert any("Skipping correction for segment 0" in message for message in progress) assert any("Skipping correction for id 1" in message for message in progress)
preserved = list((tmp_path / "work").iterdir()) preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1 assert len(preserved) == 1
skipped_path = preserved[0] / "skipped-corrections.json" skipped_path = preserved[0] / "skipped-corrections.json"
assert skipped_path.exists() assert skipped_path.exists()
diagnostics = json.loads(skipped_path.read_text(encoding="utf-8")) diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["segment_id"] == 0 assert diagnostics["skipped_corrections"][0]["id"] == 1
assert "does not match any substring" in diagnostics["skipped_corrections"][0]["reason"] assert "does not match any substring" in diagnostics["skipped_corrections"][0]["reason"]
def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_path): def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_path):
first_pass = CorrectionCandidate( first_pass = CorrectionCandidate(
segment_id=0, id=1,
original_text="Contia", original_text="Contia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
) )
second_pass = CorrectionCandidate( second_pass = CorrectionCandidate(
segment_id=0, id=1,
original_text="Chontia", original_text="Chontia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
@@ -134,21 +134,21 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
assert list((tmp_path / "work").iterdir()) == [] assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_retry_prompt_contains_only_valid_deduped_segment_ids(tmp_path): def test_pipeline_retry_prompt_contains_only_valid_deduped_ids(tmp_path):
first_bad = CorrectionCandidate( first_bad = CorrectionCandidate(
segment_id=0, id=1,
original_text="Contia", original_text="Contia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
) )
second_bad_same_segment = CorrectionCandidate( second_bad_same_segment = CorrectionCandidate(
segment_id=0, id=1,
original_text="Still wrong", original_text="Still wrong",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
) )
invalid_segment = CorrectionCandidate( invalid_segment = CorrectionCandidate(
segment_id=99, id=99,
original_text="Missing", original_text="Missing",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,
@@ -170,13 +170,13 @@ def test_pipeline_retry_prompt_contains_only_valid_deduped_segment_ids(tmp_path)
assert fake_client.calls == 2 assert fake_client.calls == 2
retry_prompt = fake_client.messages[1][1]["content"] retry_prompt = fake_client.messages[1][1]["content"]
retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1]) retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert retry_payload == [{"segment_id": 0, "original_text": "I ask Chontia."}] assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}]
assert "Retry guidance" in retry_prompt assert "Retry guidance" in retry_prompt
def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path): def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
first_pass = CorrectionCandidate( first_pass = CorrectionCandidate(
segment_id=0, id=1,
original_text="Contia", original_text="Contia",
corrected_text="Chauntea", corrected_text="Chauntea",
confidence=0.95, confidence=0.95,

View File

@@ -9,7 +9,7 @@ def test_prompt_requires_acoustically_plausible_transcription_errors():
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."} {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
] ]
""" """
) )
@@ -43,7 +43,7 @@ def test_prompt_uses_simplified_segment_payload():
transcript = parse_transcript_json( transcript = parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."} {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
] ]
""" """
) )
@@ -61,7 +61,7 @@ def test_prompt_uses_simplified_segment_payload():
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1] transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
prompt_segments = json.loads(transcript_json) prompt_segments = json.loads(transcript_json)
assert prompt_segments == [{"segment_id": 0, "original_text": "The gestures are nearby."}] assert prompt_segments == [{"id": 1, "original_text": "The gestures are nearby."}]
assert "speaker" not in prompt_segments[0] assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0] assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0] assert "end" not in prompt_segments[0]

View File

@@ -8,12 +8,13 @@ def test_valid_transcript_parses():
segments = parse_transcript_json( segments = parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Then Lyra."} {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Then Lyra."}
] ]
""" """
) )
assert len(segments) == 1 assert len(segments) == 1
assert segments[0].id == 1
assert segments[0].speaker == "Eric" assert segments[0].speaker == "Eric"
@@ -22,7 +23,7 @@ def test_transcript_rejects_extra_fields():
parse_transcript_json( parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Hi", "extra": true} {"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Hi", "extra": true}
] ]
""" """
) )
@@ -33,7 +34,64 @@ def test_transcript_rejects_bad_timestamps():
parse_transcript_json( parse_transcript_json(
""" """
[ [
{"speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"} {"id": 1, "speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_missing_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_duplicate_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 1, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_nonsequential_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 3, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_zero_or_negative_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 0, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_noninteger_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1.5, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
] ]
""" """
) )
@@ -73,4 +131,3 @@ def test_glossary_rejects_extra_fields():
extra: "nope" extra: "nope"
""" """
) )