eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Updated configuration to utilize segment ids provided in the input transcript

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-21 15:01:31 -05:00
parent 7e1a3f721a
commit ca01e46d77
13 changed files with 186 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/test_chunking.py
View File

@@ -12,7 +12,7 @@ class CountEstimator:
def _segments(count):
payload = [
{"speaker": "Eric", "start": float(i), "end": float(i + 1), "text": f"Segment {i}"}
{"id": i + 1, "speaker": "Eric", "start": float(i), "end": float(i + 1), "text": f"Segment {i}"}
for i in range(count)
]
import json
@@ -37,4 +37,3 @@ def test_chunk_transcript_allows_exact_limit():
def test_chunk_transcript_rejects_oversized_single_segment():
with pytest.raises(AuditaValidationError):
chunk_transcript(_segments(1), max_section_tokens=9, estimator=CountEstimator())

6
tests/test_config.py
View File

@@ -3,13 +3,15 @@ from pathlib import Path
import pytest
from audita.config import AuditaConfig, ConfigOverrides
from audita.config import DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR
from audita.config import DEFAULT_CONFIDENCE_THRESHOLD, DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR
from audita.errors import AuditaConfigError
def test_config_uses_defaults_with_api_key():
config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
assert config.confidence_threshold == DEFAULT_CONFIDENCE_THRESHOLD
assert config.confidence_threshold == 0.6
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
@@ -46,6 +48,7 @@ def test_config_cli_overrides_env():
},
overrides=ConfigOverrides(
max_section_tokens=100,
confidence_threshold=0.7,
max_retries=3,
glossary_max_llm_passes=2,
work_dir=Path("/tmp/cli-audita"),
@@ -53,6 +56,7 @@ def test_config_cli_overrides_env():
)
assert config.max_section_tokens == 100
assert config.confidence_threshold == 0.7
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.work_dir == Path("/tmp/cli-audita")

39
tests/test_corrections.py
View File

@@ -9,18 +9,18 @@ def _transcript():
return parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia for help."},
{"speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia for help."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
]
"""
)
def test_apply_corrections_uses_threshold_and_preserves_segment_id_order():
def test_apply_corrections_uses_threshold_and_preserves_id_order():
transcript = _transcript()
corrections = [
CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.8,
@@ -38,7 +38,7 @@ def test_apply_corrections_ignores_below_threshold():
transcript = _transcript()
corrections = [
CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.79,
@@ -54,13 +54,13 @@ def test_apply_corrections_ignores_below_threshold():
def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
transcript = _transcript()
first = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.8,
)
second = CorrectionCandidate(
segment_id=0,
id=1,
original_text="help",
corrected_text="guidance",
confidence=0.9,
@@ -75,7 +75,7 @@ def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
def test_apply_corrections_skips_missing_substring():
transcript = _transcript()
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.8,
@@ -85,15 +85,15 @@ def test_apply_corrections_skips_missing_substring():
assert result.transcript[0].text == "I ask Chontia for help."
assert len(result.skipped) == 1
assert result.skipped[0].segment_id == 0
assert result.skipped[0].id == 1
assert result.skipped[0].actual_text == "I ask Chontia for help."
assert "does not match any substring" in result.skipped[0].reason
def test_apply_corrections_skips_missing_segment_id():
def test_apply_corrections_skips_missing_id():
transcript = _transcript()
correction = CorrectionCandidate(
segment_id=99,
id=99,
original_text="Missing.",
corrected_text="Still missing.",
confidence=0.8,
@@ -103,14 +103,14 @@ def test_apply_corrections_skips_missing_segment_id():
assert [segment.text for segment in result.transcript] == ["I ask Chontia for help.", "Then Lyra."]
assert len(result.skipped) == 1
assert result.skipped[0].segment_id == 99
assert result.skipped[0].id == 99
assert "does not exist" in result.skipped[0].reason
def test_apply_corrections_skips_no_op():
transcript = _transcript()
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chontia",
confidence=0.8,
@@ -123,16 +123,16 @@ def test_apply_corrections_skips_no_op():
assert "identical" in result.skipped[0].reason
def test_apply_corrections_skips_ambiguous_repeated_substring():
def test_apply_corrections_replaces_all_repeated_substrings():
transcript = parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Bane met Bane."}
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Bane met Bane."}
]
"""
)
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Bane",
corrected_text="Bain",
confidence=0.8,
@@ -140,15 +140,14 @@ def test_apply_corrections_skips_ambiguous_repeated_substring():
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "Bane met Bane."
assert len(result.skipped) == 1
assert "multiple times" in result.skipped[0].reason
assert result.transcript[0].text == "Bain met Bain."
assert result.skipped == []
def test_apply_corrections_skips_empty_original_text():
transcript = _transcript()
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="",
corrected_text="Chauntea",
confidence=0.8,

28
tests/test_pipeline.py
View File

@@ -43,8 +43,8 @@ def _transcript():
return parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
{"speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
]
"""
)
@@ -52,7 +52,7 @@ def _transcript():
def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.95,
@@ -74,7 +74,7 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.95,
@@ -91,25 +91,25 @@ def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
)
assert revised[1].text == "I ask Chontia."
assert any("Skipping correction for segment 0" in message for message in progress)
assert any("Skipping correction for id 1" in message for message in progress)
preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1
skipped_path = preserved[0] / "skipped-corrections.json"
assert skipped_path.exists()
diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["segment_id"] == 0
assert diagnostics["skipped_corrections"][0]["id"] == 1
assert "does not match any substring" in diagnostics["skipped_corrections"][0]["reason"]
def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_path):
first_pass = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Contia",
corrected_text="Chauntea",
confidence=0.95,
)
second_pass = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.95,
@@ -134,21 +134,21 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_retry_prompt_contains_only_valid_deduped_segment_ids(tmp_path):
def test_pipeline_retry_prompt_contains_only_valid_deduped_ids(tmp_path):
first_bad = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Contia",
corrected_text="Chauntea",
confidence=0.95,
)
second_bad_same_segment = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Still wrong",
corrected_text="Chauntea",
confidence=0.95,
)
invalid_segment = CorrectionCandidate(
segment_id=99,
id=99,
original_text="Missing",
corrected_text="Chauntea",
confidence=0.95,
@@ -170,13 +170,13 @@ def test_pipeline_retry_prompt_contains_only_valid_deduped_segment_ids(tmp_path)
assert fake_client.calls == 2
retry_prompt = fake_client.messages[1][1]["content"]
retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert retry_payload == [{"segment_id": 0, "original_text": "I ask Chontia."}]
assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}]
assert "Retry guidance" in retry_prompt
def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
first_pass = CorrectionCandidate(
segment_id=0,
id=1,
original_text="Contia",
corrected_text="Chauntea",
confidence=0.95,

6
tests/test_prompts.py
View File

@@ -9,7 +9,7 @@ def test_prompt_requires_acoustically_plausible_transcription_errors():
transcript = parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
]
"""
)
@@ -43,7 +43,7 @@ def test_prompt_uses_simplified_segment_payload():
transcript = parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
]
"""
)
@@ -61,7 +61,7 @@ def test_prompt_uses_simplified_segment_payload():
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
prompt_segments = json.loads(transcript_json)
assert prompt_segments == [{"segment_id": 0, "original_text": "The gestures are nearby."}]
assert prompt_segments == [{"id": 1, "original_text": "The gestures are nearby."}]
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]

65
tests/test_validation.py
View File

@@ -8,12 +8,13 @@ def test_valid_transcript_parses():
segments = parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Then Lyra."}
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Then Lyra."}
]
"""
)
assert len(segments) == 1
assert segments[0].id == 1
assert segments[0].speaker == "Eric"
@@ -22,7 +23,7 @@ def test_transcript_rejects_extra_fields():
parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Hi", "extra": true}
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Hi", "extra": true}
]
"""
)
@@ -33,7 +34,64 @@ def test_transcript_rejects_bad_timestamps():
parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
{"id": 1, "speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_missing_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_duplicate_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 1, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_nonsequential_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 3, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_zero_or_negative_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 0, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_noninteger_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1.5, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
@@ -73,4 +131,3 @@ def test_glossary_rejects_extra_fields():
extra: "nope"
"""
)