eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Implemented a second LLM stage for grammatical review

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-21 15:42:09 -05:00
parent ca01e46d77
commit 445329de81
13 changed files with 620 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/test_cli.py
View File

@@ -16,4 +16,9 @@ def test_process_help_includes_glossary_pass_flag(capsys):
main(["process", "--help"])
assert exc.value.code == 0
assert "--glossary-max-llm-passes" in capsys.readouterr().out
output = capsys.readouterr().out
assert "--glossary-max-llm-passes" in output
assert "--grammar-max-llm-passes" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--confidence-threshold" not in output

61
tests/test_config.py
View File

@@ -3,18 +3,29 @@ from pathlib import Path
import pytest
from audita.config import AuditaConfig, ConfigOverrides
from audita.config import DEFAULT_CONFIDENCE_THRESHOLD, DEFAULT_GLOSSARY_MAX_LLM_PASSES, DEFAULT_MAX_RETRIES, DEFAULT_MAX_SECTION_TOKENS, DEFAULT_WORK_DIR
from audita.config import (
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_MAX_RETRIES,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_WORK_DIR,
)
from audita.errors import AuditaConfigError
def test_config_uses_defaults_with_api_key():
config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
assert config.confidence_threshold == DEFAULT_CONFIDENCE_THRESHOLD
assert config.confidence_threshold == 0.6
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.glossary_confidence_threshold == 0.6
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == 0.6
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
assert config.grammar_max_llm_passes == DEFAULT_GRAMMAR_MAX_LLM_PASSES
assert config.work_dir == Path(DEFAULT_WORK_DIR)
@@ -23,17 +34,21 @@ def test_config_env_overrides_defaults():
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_WORK_DIR": "/tmp/custom-audita",
}
)
assert config.max_section_tokens == 42
assert config.confidence_threshold == 0.9
assert config.glossary_confidence_threshold == 0.9
assert config.grammar_confidence_threshold == 0.7
assert config.max_retries == 5
assert config.glossary_max_llm_passes == 7
assert config.grammar_max_llm_passes == 4
assert config.work_dir == Path("/tmp/custom-audita")
@@ -44,21 +59,26 @@ def test_config_cli_overrides_env():
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_WORK_DIR": "/tmp/env-audita",
},
overrides=ConfigOverrides(
max_section_tokens=100,
confidence_threshold=0.7,
glossary_confidence_threshold=0.7,
grammar_confidence_threshold=0.65,
max_retries=3,
glossary_max_llm_passes=2,
grammar_max_llm_passes=3,
work_dir=Path("/tmp/cli-audita"),
),
)
assert config.max_section_tokens == 100
assert config.confidence_threshold == 0.7
assert config.glossary_confidence_threshold == 0.7
assert config.grammar_confidence_threshold == 0.65
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.grammar_max_llm_passes == 3
assert config.work_dir == Path("/tmp/cli-audita")
@@ -79,3 +99,30 @@ def test_config_rejects_invalid_glossary_pass_count():
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_grammar_pass_count():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_stage_thresholds():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "-0.1"}
)
def test_legacy_confidence_threshold_env_is_ignored():
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_CONFIDENCE_THRESHOLD": "0.9"}
)
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD

32
tests/test_corrections.py
View File

@@ -144,6 +144,33 @@ def test_apply_corrections_replaces_all_repeated_substrings():
assert result.skipped == []
def test_apply_corrections_requires_unique_match_when_configured():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there"}
]
"""
)
correction = CorrectionCandidate(
id=1,
original_text="there",
corrected_text="their",
confidence=0.8,
)
result = apply_corrections(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
)
assert result.transcript[0].text == "there and there"
assert len(result.skipped) == 1
assert "more than once" in result.skipped[0].reason
def test_apply_corrections_skips_empty_original_text():
transcript = _transcript()
correction = CorrectionCandidate(
@@ -163,3 +190,8 @@ def test_apply_corrections_skips_empty_original_text():
def test_apply_corrections_rejects_invalid_threshold():
with pytest.raises(AuditaValidationError):
apply_corrections(_transcript(), [], confidence_threshold=1.1)
def test_apply_corrections_rejects_invalid_replacement_mode():
with pytest.raises(AuditaValidationError):
apply_corrections(_transcript(), [], confidence_threshold=0.8, replacement_mode="unknown")

184
tests/test_pipeline.py
View File

@@ -17,13 +17,15 @@ class FakeLLMClient:
return self.responses.pop(0)
def _config(tmp_path, glossary_max_llm_passes=3):
def _config(tmp_path, glossary_max_llm_passes=3, grammar_max_llm_passes=3):
return AuditaConfig(
api_key="key",
max_section_tokens=16000,
confidence_threshold=0.8,
glossary_confidence_threshold=0.8,
grammar_confidence_threshold=0.8,
max_retries=3,
glossary_max_llm_passes=glossary_max_llm_passes,
grammar_max_llm_passes=grammar_max_llm_passes,
work_dir=tmp_path / "work",
)
@@ -57,7 +59,12 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient([CorrectionSet(corrections=[correction])])
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
revised = process_transcript(
_transcript(),
@@ -68,18 +75,23 @@ def test_pipeline_processes_with_fake_llm_and_cleans_work_dir(tmp_path):
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert fake_client.calls == 1
assert fake_client.calls == 2
assert list((tmp_path / "work").iterdir()) == []
def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
def test_pipeline_skips_bad_glossary_correction_and_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.95,
)
fake_client = FakeLLMClient([CorrectionSet(corrections=[correction])])
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[correction]),
CorrectionSet(corrections=[]),
]
)
progress = []
revised = process_transcript(
@@ -91,12 +103,13 @@ def test_pipeline_skips_bad_correction_and_preserves_diagnostics(tmp_path):
)
assert revised[1].text == "I ask Chontia."
assert any("Skipping correction for id 1" in message for message in progress)
assert any("Skipping glossary correction for id 1" in message for message in progress)
preserved = list((tmp_path / "work").iterdir())
assert len(preserved) == 1
skipped_path = preserved[0] / "skipped-corrections.json"
assert skipped_path.exists()
diagnostics = json.loads(skipped_path.read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "glossary"
assert diagnostics["skipped_corrections"][0]["id"] == 1
assert "does not match any substring" in diagnostics["skipped_corrections"][0]["reason"]
@@ -118,6 +131,7 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
[
CorrectionSet(corrections=[first_pass]),
CorrectionSet(corrections=[second_pass]),
CorrectionSet(corrections=[]),
]
)
@@ -128,7 +142,7 @@ def test_pipeline_retries_skipped_segment_and_cleans_work_dir_when_fixed(tmp_pat
llm_client=fake_client,
)
assert fake_client.calls == 2
assert fake_client.calls == 3
assert [segment.speaker for segment in revised] == ["Mike", "Eric"]
assert revised[1].text == "I ask Chauntea."
assert list((tmp_path / "work").iterdir()) == []
@@ -157,6 +171,7 @@ def test_pipeline_retry_prompt_contains_only_valid_deduped_ids(tmp_path):
[
CorrectionSet(corrections=[first_bad, second_bad_same_segment, invalid_segment]),
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[]),
]
)
@@ -167,14 +182,14 @@ def test_pipeline_retry_prompt_contains_only_valid_deduped_ids(tmp_path):
llm_client=fake_client,
)
assert fake_client.calls == 2
assert fake_client.calls == 3
retry_prompt = fake_client.messages[1][1]["content"]
retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert retry_payload == [{"id": 1, "original_text": "I ask Chontia."}]
assert "Retry guidance" in retry_prompt
def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
def test_pipeline_writes_stage_metadata_for_unresolved_retries(tmp_path):
first_pass = CorrectionCandidate(
id=1,
original_text="Contia",
@@ -185,6 +200,7 @@ def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
[
CorrectionSet(corrections=[first_pass]),
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[]),
]
)
@@ -197,8 +213,150 @@ def test_pipeline_writes_pass_metadata_for_unresolved_retries(tmp_path):
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
assert (run_dirs[0] / "glossary" / "pass-0001").exists()
assert (run_dirs[0] / "grammar" / "pass-0001").exists()
metadata = json.loads((run_dirs[0] / "metadata.json").read_text(encoding="utf-8"))
assert metadata["glossary_max_llm_passes"] == 2
assert [item["pass_number"] for item in metadata["passes"]] == [1, 2]
assert metadata["passes"][0]["retry_segment_count"] == 1
assert metadata["passes"][1]["retry_pass"] is True
assert metadata["grammar_max_llm_passes"] == 3
assert metadata["glossary_confidence_threshold"] == 0.8
assert metadata["grammar_confidence_threshold"] == 0.8
assert [item["stage"] for item in metadata["stages"]] == ["glossary", "grammar"]
assert [item["pass_number"] for item in metadata["stages"][0]["passes"]] == [1, 2]
assert metadata["stages"][0]["passes"][0]["retry_segment_count"] == 1
assert metadata["stages"][0]["passes"][1]["retry_pass"] is True
def test_grammar_stage_runs_after_glossary_and_sees_corrected_text(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "i ask Chontia."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
]
"""
)
glossary_correction = CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.95,
)
grammar_correction = CorrectionCandidate(
id=1,
original_text="i",
corrected_text="I",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[glossary_correction]),
CorrectionSet(corrections=[grammar_correction]),
]
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path),
llm_client=fake_client,
)
grammar_prompt = fake_client.messages[1][1]["content"]
grammar_payload = json.loads(grammar_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert grammar_payload[0]["original_text"] == "i ask Chauntea."
assert revised[1].text == "I ask Chauntea."
def test_grammar_stage_retries_repeated_span_and_applies_unique_retry(tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there."}
]
"""
)
repeated_span = CorrectionCandidate(
id=1,
original_text="there",
corrected_text="their",
confidence=0.95,
)
unique_retry = CorrectionCandidate(
id=1,
original_text="there and there",
corrected_text="their and there",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[repeated_span]),
CorrectionSet(corrections=[unique_retry]),
]
)
revised = process_transcript(
transcript,
_glossary(),
_config(tmp_path, grammar_max_llm_passes=2),
llm_client=fake_client,
)
assert fake_client.calls == 3
assert revised[0].text == "their and there."
retry_prompt = fake_client.messages[2][1]["content"]
retry_payload = json.loads(retry_prompt.split("Transcript section:\n", maxsplit=1)[1])
assert retry_payload == [{"id": 1, "original_text": "there and there."}]
def test_below_threshold_grammar_corrections_are_not_retried(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="I",
corrected_text="i",
confidence=0.7,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[correction]),
]
)
revised = process_transcript(
_transcript(),
_glossary(),
_config(tmp_path, grammar_max_llm_passes=3),
llm_client=fake_client,
)
assert fake_client.calls == 2
assert revised[1].text == "I ask Chontia."
def test_unresolved_grammar_skip_preserves_diagnostics(tmp_path):
correction = CorrectionCandidate(
id=1,
original_text="a",
corrected_text="A",
confidence=0.95,
)
fake_client = FakeLLMClient(
[
CorrectionSet(corrections=[]),
CorrectionSet(corrections=[correction]),
]
)
process_transcript(
_transcript(),
_glossary(),
_config(tmp_path, grammar_max_llm_passes=1),
llm_client=fake_client,
)
run_dirs = list((tmp_path / "work").iterdir())
assert len(run_dirs) == 1
diagnostics = json.loads((run_dirs[0] / "skipped-corrections.json").read_text(encoding="utf-8"))
assert diagnostics["skipped_corrections"][0]["stage"] == "grammar"
assert "more than once" in diagnostics["skipped_corrections"][0]["reason"]

61
tests/test_prompts.py
View File

@@ -1,7 +1,7 @@
import json
from audita.chunking import chunk_transcript
from audita.prompts import build_glossary_correction_messages
from audita.prompts import build_glossary_correction_messages, build_grammar_correction_messages
from audita.schemas import parse_glossary_yaml, parse_transcript_json
@@ -65,3 +65,62 @@ def test_prompt_uses_simplified_segment_payload():
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]
def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_grammar_correction_messages(section, glossary)
prompt_text = "\n".join(message["content"] for message in messages)
assert "capitalization" in prompt_text
assert "commas, periods, em dashes, and ellipses" in prompt_text
assert "homophone fixes" in prompt_text
assert "spelling fixes" in prompt_text
assert "Do not paraphrase" in prompt_text
assert "protected vocabulary" in prompt_text
assert "appears exactly once" in prompt_text
assert "Do not return speaker, start, or end fields" in prompt_text
def test_grammar_prompt_uses_simplified_segment_payload():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_grammar_correction_messages(section, glossary)
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
prompt_segments = json.loads(transcript_json)
assert prompt_segments == [{"id": 1, "original_text": "then lyra went their"}]
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]