eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Removed historical prototype code

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-28 22:35:32 -05:00
parent e71e5b8faf
commit 177e45f1fd
30 changed files with 0 additions and 6055 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/audita_prototype/__init__.py
View File

@@ -1 +0,0 @@
"""Archived prototype regression suite."""

39
tests/audita_prototype/test_chunking.py
View File

@@ -1,39 +0,0 @@
import pytest
from audita_prototype.chunking import chunk_transcript
from audita_prototype.errors import AuditaValidationError
from audita_prototype.schemas import parse_transcript_json
class CountEstimator:
def estimate_json(self, value):
return len(value) * 10
def _segments(count):
payload = [
{"id": i + 1, "speaker": "Eric", "start": float(i), "end": float(i + 1), "text": f"Segment {i}"}
for i in range(count)
]
import json
return parse_transcript_json(json.dumps(payload))
def test_chunk_transcript_splits_on_segment_boundaries():
sections = chunk_transcript(_segments(5), max_section_tokens=20, estimator=CountEstimator())
assert [len(section.segments) for section in sections] == [2, 2, 1]
assert [section.start_index for section in sections] == [0, 2, 4]
def test_chunk_transcript_allows_exact_limit():
sections = chunk_transcript(_segments(2), max_section_tokens=20, estimator=CountEstimator())
assert len(sections) == 1
assert len(sections[0].segments) == 2
def test_chunk_transcript_rejects_oversized_single_segment():
with pytest.raises(AuditaValidationError):
chunk_transcript(_segments(1), max_section_tokens=9, estimator=CountEstimator())

87
tests/audita_prototype/test_cli.py
View File

@@ -1,87 +0,0 @@
import pytest
from audita_prototype.cli import main
from audita_prototype.reporting import ProcessResult, RunReport
from audita_prototype.schemas import parse_transcript_json
def test_cli_help_uses_audita_program_name(capsys):
with pytest.raises(SystemExit) as exc:
main(["--help"])
assert exc.value.code == 0
assert capsys.readouterr().out.startswith("usage: audita ")
def test_process_help_includes_glossary_pass_flag(capsys):
with pytest.raises(SystemExit) as exc:
main(["process", "--help"])
assert exc.value.code == 0
output = capsys.readouterr().out
assert "--report-json" in output
assert "--glossary-max-llm-passes" in output
assert "--grammar-max-llm-passes" in output
assert "--glossary-confidence-threshold" in output
assert "--grammar-confidence-threshold" in output
assert "--grammar-validation-enabled" in output
assert "--grammar-validation-confidence-threshold" in output
assert "--grammar-spoken-form-validation-confidence-threshold" in output
assert "--work-dir-retention" in output
assert "--normalize-max-segment-gap" in output
assert "--normalize-ellipsis-gap" in output
assert "--normalize-max-segment-duration" in output
assert "--normalize-max-segment-tokens" in output
assert "--confidence-threshold" not in output
def test_cli_process_writes_report_json(monkeypatch, tmp_path):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
stages=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
monkeypatch.setattr("audita_prototype.cli.AuditaConfig.from_sources", lambda overrides=None: object())
monkeypatch.setattr("audita_prototype.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita_prototype.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita_prototype.cli.process_transcript_result", lambda *args, **kwargs: result)
output_path = tmp_path / "out.json"
report_path = tmp_path / "report.json"
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--output",
str(output_path),
"--report-json",
str(report_path),
]
)
assert exit_code == 0
assert report_path.exists()

236
tests/audita_prototype/test_config.py
View File

@@ -1,236 +0,0 @@
from pathlib import Path
import pytest
from audita_prototype.config import AuditaConfig, ConfigOverrides
from audita_prototype.config import (
DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD,
DEFAULT_GLOSSARY_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_MAX_LLM_PASSES,
DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD,
DEFAULT_GRAMMAR_VALIDATION_ENABLED,
DEFAULT_MAX_RETRIES,
DEFAULT_MAX_SECTION_TOKENS,
DEFAULT_NORMALIZE_ELLIPSIS_GAP,
DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION,
DEFAULT_NORMALIZE_MAX_SEGMENT_GAP,
DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS,
DEFAULT_WORK_DIR,
DEFAULT_WORK_DIR_RETENTION,
)
from audita_prototype.errors import AuditaConfigError
def test_config_uses_defaults_with_api_key():
config = AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key"})
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.glossary_confidence_threshold == 0.8
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == 0.8
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.max_retries == DEFAULT_MAX_RETRIES
assert config.glossary_max_llm_passes == DEFAULT_GLOSSARY_MAX_LLM_PASSES
assert config.grammar_max_llm_passes == DEFAULT_GRAMMAR_MAX_LLM_PASSES
assert config.grammar_validation_enabled == DEFAULT_GRAMMAR_VALIDATION_ENABLED
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == DEFAULT_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD
assert config.grammar_validation_confidence_threshold == 0.8
assert (
config.grammar_spoken_form_validation_confidence_threshold
== DEFAULT_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD
)
assert config.grammar_spoken_form_validation_confidence_threshold == 0.8
assert config.normalize_max_segment_gap == DEFAULT_NORMALIZE_MAX_SEGMENT_GAP
assert config.normalize_max_segment_gap == 4.0
assert config.normalize_ellipsis_gap == DEFAULT_NORMALIZE_ELLIPSIS_GAP
assert config.normalize_max_segment_duration == DEFAULT_NORMALIZE_MAX_SEGMENT_DURATION
assert config.normalize_max_segment_duration == 60.0
assert config.normalize_max_segment_tokens == DEFAULT_NORMALIZE_MAX_SEGMENT_TOKENS
assert config.normalize_max_segment_tokens == 2048
assert config.work_dir == Path(DEFAULT_WORK_DIR)
assert config.work_dir_retention == DEFAULT_WORK_DIR_RETENTION
assert config.work_dir_retention == "auto"
def test_config_env_overrides_defaults():
config = AuditaConfig.from_sources(
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "0.9",
"AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "0.7",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "4",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/custom-audita",
"AUDITA_WORK_DIR_RETENTION": "always",
}
)
assert config.max_section_tokens == 42
assert config.glossary_confidence_threshold == 0.9
assert config.grammar_confidence_threshold == 0.7
assert config.max_retries == 5
assert config.glossary_max_llm_passes == 7
assert config.grammar_max_llm_passes == 4
assert config.grammar_validation_enabled is False
assert config.grammar_validation_confidence_threshold == 0.91
assert config.grammar_spoken_form_validation_confidence_threshold == 0.87
assert config.normalize_max_segment_gap == 4.5
assert config.normalize_ellipsis_gap == 1.5
assert config.normalize_max_segment_duration == 45.0
assert config.normalize_max_segment_tokens == 512
assert config.work_dir == Path("/tmp/custom-audita")
assert config.work_dir_retention == "always"
def test_config_cli_overrides_env():
config = AuditaConfig.from_sources(
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_MAX_SECTION_TOKENS": "42",
"AUDITA_MAX_RETRIES": "5",
"AUDITA_GLOSSARY_MAX_LLM_PASSES": "7",
"AUDITA_GRAMMAR_MAX_LLM_PASSES": "6",
"AUDITA_GRAMMAR_VALIDATION_ENABLED": "false",
"AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "0.91",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "0.87",
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "4.5",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "1.5",
"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "45.0",
"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "512",
"AUDITA_WORK_DIR": "/tmp/env-audita",
"AUDITA_WORK_DIR_RETENTION": "always",
},
overrides=ConfigOverrides(
max_section_tokens=100,
glossary_confidence_threshold=0.7,
grammar_confidence_threshold=0.65,
max_retries=3,
glossary_max_llm_passes=2,
grammar_max_llm_passes=3,
grammar_validation_enabled=True,
grammar_validation_confidence_threshold=0.75,
grammar_spoken_form_validation_confidence_threshold=0.72,
normalize_max_segment_gap=3.0,
normalize_ellipsis_gap=1.0,
normalize_max_segment_duration=30.0,
normalize_max_segment_tokens=256,
work_dir=Path("/tmp/cli-audita"),
work_dir_retention="never",
),
)
assert config.max_section_tokens == 100
assert config.glossary_confidence_threshold == 0.7
assert config.grammar_confidence_threshold == 0.65
assert config.max_retries == 3
assert config.glossary_max_llm_passes == 2
assert config.grammar_max_llm_passes == 3
assert config.grammar_validation_enabled is True
assert config.grammar_validation_confidence_threshold == 0.75
assert config.grammar_spoken_form_validation_confidence_threshold == 0.72
assert config.normalize_max_segment_gap == 3.0
assert config.normalize_ellipsis_gap == 1.0
assert config.normalize_max_segment_duration == 30.0
assert config.normalize_max_segment_tokens == 256
assert config.work_dir == Path("/tmp/cli-audita")
assert config.work_dir_retention == "never"
def test_config_requires_api_key():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={})
def test_config_rejects_bad_env_int():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_MAX_SECTION_TOKENS": "many"}
)
def test_config_rejects_invalid_glossary_pass_count():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_grammar_pass_count():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_MAX_LLM_PASSES": "0"}
)
def test_config_rejects_invalid_stage_thresholds():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GLOSSARY_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_CONFIDENCE_THRESHOLD": "-0.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_CONFIDENCE_THRESHOLD": "1.1"}
)
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={
"OPENROUTER_API_KEY": "key",
"AUDITA_GRAMMAR_SPOKEN_FORM_VALIDATION_CONFIDENCE_THRESHOLD": "-0.1",
}
)
def test_config_rejects_invalid_grammar_validation_enabled():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_GRAMMAR_VALIDATION_ENABLED": "maybe"}
)
def test_config_rejects_invalid_work_dir_retention():
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_WORK_DIR_RETENTION": "sometimes"}
)
def test_legacy_confidence_threshold_env_is_ignored():
config = AuditaConfig.from_sources(
env={"OPENROUTER_API_KEY": "key", "AUDITA_CONFIDENCE_THRESHOLD": "0.9"}
)
assert config.glossary_confidence_threshold == DEFAULT_GLOSSARY_CONFIDENCE_THRESHOLD
assert config.grammar_confidence_threshold == DEFAULT_GRAMMAR_CONFIDENCE_THRESHOLD
def test_config_rejects_invalid_normalization_values():
invalid_envs = [
{"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "-1"},
{"AUDITA_NORMALIZE_ELLIPSIS_GAP": "-1"},
{
"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "1",
"AUDITA_NORMALIZE_ELLIPSIS_GAP": "2",
},
{"AUDITA_NORMALIZE_MAX_SEGMENT_DURATION": "0"},
{"AUDITA_NORMALIZE_MAX_SEGMENT_TOKENS": "0"},
{"AUDITA_NORMALIZE_MAX_SEGMENT_GAP": "nan"},
]
for env in invalid_envs:
with pytest.raises(AuditaConfigError):
AuditaConfig.from_sources(env={"OPENROUTER_API_KEY": "key", **env})

294
tests/audita_prototype/test_corrections.py
View File

@@ -1,294 +0,0 @@
import pytest
from audita_prototype.corrections import apply_corrections
from audita_prototype.errors import AuditaValidationError
from audita_prototype.protection import ProtectedVocabulary
from audita_prototype.schemas import CorrectionCandidate, parse_glossary_yaml, parse_transcript_json
def _transcript():
return parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 10.0, "end": 11.0, "text": "I ask Chontia for help."},
{"id": 2, "speaker": "Mike", "start": 0.0, "end": 1.0, "text": "Then Lyra."}
]
"""
)
def test_apply_corrections_uses_threshold_and_preserves_id_order():
transcript = _transcript()
corrections = [
CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.8,
)
]
result = apply_corrections(transcript, corrections, confidence_threshold=0.8)
assert [segment.speaker for segment in result.transcript] == ["Eric", "Mike"]
assert result.transcript[0].text == "I ask Chauntea for help."
assert result.skipped == []
assert len(result.applied_corrections) == 1
assert result.applied_corrections[0].segment_text_before == "I ask Chontia for help."
assert result.applied_corrections[0].segment_text_after == "I ask Chauntea for help."
def test_apply_corrections_ignores_below_threshold():
transcript = _transcript()
corrections = [
CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.79,
)
]
result = apply_corrections(transcript, corrections, confidence_threshold=0.8)
assert result.transcript[0].text == "I ask Chontia for help."
assert result.skipped == []
assert result.ignored_ids == [1]
assert len(result.ignored) == 1
assert result.ignored[0].id == 1
assert result.ignored[0].reason == "correction confidence below threshold"
def test_apply_corrections_allows_multiple_distinct_spans_in_one_segment():
transcript = _transcript()
first = CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chauntea",
confidence=0.8,
)
second = CorrectionCandidate(
id=1,
original_text="help",
corrected_text="guidance",
confidence=0.9,
)
result = apply_corrections(transcript, [first, second], confidence_threshold=0.8)
assert result.transcript[0].text == "I ask Chauntea for guidance."
assert result.skipped == []
assert len(result.applied_corrections) == 2
def test_apply_corrections_skips_missing_substring():
transcript = _transcript()
correction = CorrectionCandidate(
id=1,
original_text="Different text.",
corrected_text="Chauntea",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "I ask Chontia for help."
assert len(result.skipped) == 1
assert result.skipped[0].id == 1
assert result.skipped[0].actual_text == "I ask Chontia for help."
assert "does not match any substring" in result.skipped[0].reason
def test_apply_corrections_skips_missing_id():
transcript = _transcript()
correction = CorrectionCandidate(
id=99,
original_text="Missing.",
corrected_text="Still missing.",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert [segment.text for segment in result.transcript] == ["I ask Chontia for help.", "Then Lyra."]
assert len(result.skipped) == 1
assert result.skipped[0].id == 99
assert "does not exist" in result.skipped[0].reason
def test_apply_corrections_skips_no_op():
transcript = _transcript()
correction = CorrectionCandidate(
id=1,
original_text="Chontia",
corrected_text="Chontia",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "I ask Chontia for help."
assert len(result.skipped) == 1
assert "identical" in result.skipped[0].reason
def test_apply_corrections_replaces_all_repeated_substrings():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Bane met Bane."}
]
"""
)
correction = CorrectionCandidate(
id=1,
original_text="Bane",
corrected_text="Bain",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "Bain met Bain."
assert result.skipped == []
def test_apply_corrections_requires_unique_match_when_configured():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "there and there"}
]
"""
)
correction = CorrectionCandidate(
id=1,
original_text="there",
corrected_text="their",
confidence=0.8,
)
result = apply_corrections(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
)
assert result.transcript[0].text == "there and there"
assert len(result.skipped) == 1
assert "more than once" in result.skipped[0].reason
def test_apply_corrections_skips_when_guard_rejects_replacement():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hrank moves."}
]
"""
)
correction = CorrectionCandidate(
id=1,
original_text="Hrank",
corrected_text="Frank",
confidence=0.8,
)
result = apply_corrections(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
correction_guard=lambda before, after: "protected term changed" if before != after else None,
)
assert result.transcript[0].text == "Hrank moves."
assert len(result.skipped) == 1
assert result.skipped[0].reason == "protected term changed"
assert result.skipped[0].actual_text == "Hrank moves."
def test_apply_corrections_guards_only_replacement_span():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "You have to keep it bind. Svend sees the jesters."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Svend"
category: pc
summary: "Svend is a player character."
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
correction = CorrectionCandidate(
id=1,
original_text="keep it bind",
corrected_text="keep in mind",
confidence=0.8,
)
result = apply_corrections(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
correction_guard=ProtectedVocabulary.from_glossary(glossary).violation_reason,
)
assert result.transcript[0].text == "You have to keep in mind. Svend sees the jesters."
assert result.skipped == []
def test_apply_corrections_without_guard_remains_permissive():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Frank moves."}
]
"""
)
correction = CorrectionCandidate(
id=1,
original_text="Frank",
corrected_text="Hrank",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "Hrank moves."
assert result.skipped == []
def test_apply_corrections_skips_empty_original_text():
transcript = _transcript()
correction = CorrectionCandidate(
id=1,
original_text="",
corrected_text="Chauntea",
confidence=0.8,
)
result = apply_corrections(transcript, [correction], confidence_threshold=0.8)
assert result.transcript[0].text == "I ask Chontia for help."
assert len(result.skipped) == 1
assert "empty" in result.skipped[0].reason
def test_apply_corrections_rejects_invalid_threshold():
with pytest.raises(AuditaValidationError):
apply_corrections(_transcript(), [], confidence_threshold=1.1)
def test_apply_corrections_rejects_invalid_replacement_mode():
with pytest.raises(AuditaValidationError):
apply_corrections(_transcript(), [], confidence_threshold=0.8, replacement_mode="unknown")

33
tests/audita_prototype/test_launcher.py
View File

@@ -1,33 +0,0 @@
import os
import shutil
import subprocess
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
def test_prototype_package_is_importable():
package_root = ROOT / "src" / "audita_prototype"
assert package_root.is_dir()
assert (package_root / "__main__.py").is_file()
def test_prototype_module_help_smoke():
env = os.environ.copy()
env["PYTHONPATH"] = str(ROOT / "src")
result = subprocess.run(
[sys.executable, "-m", "audita_prototype", "--help"],
cwd=ROOT,
text=True,
capture_output=True,
env=env,
check=False,
)
assert result.returncode == 0
assert result.stdout.startswith("usage: audita ")

153
tests/audita_prototype/test_normalization.py
View File

@@ -1,153 +0,0 @@
from audita_prototype.normalization import normalize_transcript
from audita_prototype.schemas import parse_source_transcript_json
class WordEstimator:
def estimate_json(self, value):
return len(value[0]["original_text"].split())
def _normalize(raw, **overrides):
defaults = {
"max_segment_gap": 5.0,
"ellipsis_gap": 2.0,
"max_segment_duration": 60.0,
"max_segment_tokens": 2048,
}
defaults.update(overrides)
return normalize_transcript(parse_source_transcript_json(raw), **defaults)
def test_same_speaker_short_gap_merges_with_space():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "there"}
]
"""
)
assert len(result.transcript) == 1
assert result.transcript[0].id == 1
assert result.transcript[0].text == "Hello there"
assert result.transcript[0].start == 0.0
assert result.transcript[0].end == 3.0
assert result.summary.merge_count == 1
def test_same_speaker_larger_allowed_gap_merges_with_ellipsis():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 4.0, "end": 5.0, "text": "there"}
]
"""
)
assert result.transcript[0].text == "Hello ... there"
def test_different_speakers_do_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Mike", "start": 1.5, "end": 2.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
assert result.summary.merge_count == 0
def test_gap_above_max_does_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hello"},
{"speaker": "Eric", "start": 7.0, "end": 8.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_overlapping_segments_do_not_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 2.0, "text": "Hello"},
{"speaker": "Eric", "start": 1.5, "end": 3.0, "text": "there"}
]
"""
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_max_duration_prevents_merge():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 40.0, "text": "Hello"},
{"speaker": "Eric", "start": 45.0, "end": 50.0, "text": "there"}
]
""",
max_segment_duration=45.0,
)
assert [segment.text for segment in result.transcript] == ["Hello", "there"]
def test_max_token_limit_prevents_merge():
segments = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "one two"},
{"speaker": "Eric", "start": 2.0, "end": 3.0, "text": "three four"}
]
"""
)
result = normalize_transcript(
segments,
max_segment_gap=5.0,
ellipsis_gap=2.0,
max_segment_duration=60.0,
max_segment_tokens=3,
estimator=WordEstimator(),
)
assert [segment.text for segment in result.transcript] == ["one two", "three four"]
def test_shortest_gap_merges_first():
result = _normalize(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "A"},
{"speaker": "Eric", "start": 3.0, "end": 4.0, "text": "B"},
{"speaker": "Eric", "start": 4.5, "end": 5.0, "text": "C"}
]
""",
max_segment_duration=4.0,
)
assert [segment.text for segment in result.transcript] == ["A", "B C"]
def test_fresh_ids_are_assigned_chronologically_and_source_ids_are_discarded():
result = _normalize(
"""
[
{"id": 99, "speaker": "Mike", "start": 10.0, "end": 11.0, "text": "Second"},
{"id": 42, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "First"}
]
"""
)
assert [(segment.id, segment.text) for segment in result.transcript] == [(1, "First"), (2, "Second")]

1557
tests/audita_prototype/test_pipeline.py
View File

File diff suppressed because it is too large Load Diff

222
tests/audita_prototype/test_prompts.py
View File

@@ -1,222 +0,0 @@
import json
from audita_prototype.chunking import chunk_transcript
from audita_prototype.prompts import (
build_glossary_correction_messages,
build_grammar_correction_messages,
build_grammar_spoken_form_validation_messages,
build_grammar_validation_messages,
)
from audita_prototype.schemas import parse_glossary_yaml, parse_transcript_json
def test_prompt_requires_acoustically_plausible_transcription_errors():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a local faction."
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_glossary_correction_messages(section, glossary)
prompt_text = "\n".join(message["content"] for message in messages)
assert "acoustically plausible" in prompt_text
assert "phonetically or acoustically similar" in prompt_text
assert '"gestures" to "Jesters"' in prompt_text
assert '"Lyra" to "Jesters"' in prompt_text
assert "should be omitted" in prompt_text
assert "glossary names and aliases already present in the transcript as protected spellings" in prompt_text
assert "Do not replace, Anglicize, normalize, lowercase" in prompt_text
assert "Preserve canonical glossary capitalization" in prompt_text
assert "exact text span that needs replacement" in prompt_text
assert "replacement text for that span" in prompt_text
assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text
def test_prompt_uses_simplified_segment_payload():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a local faction."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_glossary_correction_messages(section, glossary)
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
prompt_segments = json.loads(transcript_json)
assert prompt_segments == [{"id": 1, "original_text": "The gestures are nearby."}]
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]
def test_prompts_do_not_include_inferred_plurals():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Godfrey"
aliases:
- "Jester"
category: npc
summary: "Godfrey is an NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
glossary_messages = build_glossary_correction_messages(section, glossary)
glossary_json = glossary_messages[1]["content"].split("Glossary:\n", maxsplit=1)[1].split(
"\n\nTranscript section:",
maxsplit=1,
)[0]
grammar_messages = build_grammar_correction_messages(section, glossary)
grammar_json = grammar_messages[1]["content"].split("Protected glossary/context:\n", maxsplit=1)[1].split(
"\n\nTranscript section:",
maxsplit=1,
)[0]
for prompt_glossary in (json.loads(glossary_json), json.loads(grammar_json)):
entry = prompt_glossary["glossary"][0]
assert "plural" not in entry
assert "Godfreys" not in json.dumps(prompt_glossary)
assert "Jesters" not in json.dumps(prompt_glossary)
def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_grammar_correction_messages(section, glossary)
prompt_text = "\n".join(message["content"] for message in messages)
assert "capitalization" in prompt_text
assert "commas, periods, em dashes, and ellipses" in prompt_text
assert "homophone fixes" in prompt_text
assert "spelling fixes" in prompt_text
assert "Do not paraphrase" in prompt_text
assert "glossary names and aliases as protected spellings" in prompt_text
assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
assert "that already appear correctly in the transcript" in prompt_text
assert "Preserve canonical glossary capitalization" in prompt_text
assert "appears exactly once" in prompt_text
assert "Do not return speaker, start, or end fields" in prompt_text
def test_grammar_prompt_uses_simplified_segment_payload():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
messages = build_grammar_correction_messages(section, glossary)
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
prompt_segments = json.loads(transcript_json)
assert prompt_segments == [{"id": 1, "original_text": "then lyra went their"}]
assert "speaker" not in prompt_segments[0]
assert "start" not in prompt_segments[0]
assert "end" not in prompt_segments[0]
def test_grammar_validation_prompt_rejects_semantic_changes():
messages = build_grammar_validation_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "He became visible.",
"corrected_segment_text": "He became invisible.",
"original_text": "visible",
"corrected_text": "invisible",
}
]
)
prompt_text = "\n".join(message["content"] for message in messages)
assert "preserves meaning" in prompt_text
assert "became visible" in prompt_text
assert "became invisible" in prompt_text
assert "reverses the meaning" in prompt_text
assert "do not try to rescue likely homophone or transcription fixes" in prompt_text
assert "handled in a separate spoken-form validation step" in prompt_text
assert "correction_index" in prompt_text
assert "is_meaning_preserving" in prompt_text
def test_grammar_spoken_form_validation_prompt_allows_homophone_rescue():
messages = build_grammar_spoken_form_validation_messages(
[
{
"correction_index": 0,
"id": 1,
"original_segment_text": "ChatGPT still can't really do that with a dam.",
"corrected_segment_text": "ChatGPT still can't really do that with a damn.",
"original_text": "dam",
"corrected_text": "damn",
}
]
)
prompt_text = "\n".join(message["content"] for message in messages)
assert "likely homophone, spoken-form, or transcription fix" in prompt_text
assert '"dam" to "damn"' in prompt_text
assert '"became visible" to "became invisible"' in prompt_text
assert "is_likely_spoken_form_correction" in prompt_text

191
tests/audita_prototype/test_protection.py
View File

@@ -1,191 +0,0 @@
from audita_prototype.protection import ProtectedVocabulary
from audita_prototype.schemas import parse_glossary_yaml
def _vocabulary():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Hrank"
aliases:
- "Greenfield"
category: pc
summary: "Hrank Greenfield is a player character."
- name: "Popov"
category: npc
summary: "Popov is an allied NPC."
- name: "Jesters"
aliases:
- "Jester"
category: faction
summary: "The Jesters are a faction."
- name: "Svend"
category: pc
summary: "Svend is a player character."
- name: "Godfrey"
category: npc
summary: "Godfrey is an NPC."
- name: "Lyra"
category: npc
summary: "Lyra is an NPC."
- name: "Loviator"
category: deity
summary: "Loviator is a deity."
"""
)
return ProtectedVocabulary.from_glossary(glossary)
def test_protection_blocks_replacing_protected_term():
vocabulary = _vocabulary()
assert (
vocabulary.violation_reason("Hrank moves.", "Frank moves.")
== "correction changes protected glossary term usage"
)
def test_protection_blocks_replacing_possessive_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Popov's exhausted.", "Pawpaw's exhausted.") is not None
def test_protection_blocks_lowercasing_protected_term():
vocabulary = _vocabulary()
assert (
vocabulary.violation_reason("Hrank moves.", "hrank moves.")
== "correction changes protected glossary term capitalization"
)
def test_protection_blocks_noncanonical_uppercase_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Popov moves.", "POPOV moves.") is not None
def test_protection_allows_canonical_capitalization():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("hrank moves.", "Hrank moves.") is None
def test_protection_allows_unchanged_noncanonical_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("jesters advance.", "jesters advance.") is None
def test_protection_allows_correction_toward_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Pawpaw moves.", "Popov moves.") is None
assert vocabulary.violation_reason("gestures", "Jesters") is None
assert vocabulary.violation_reason("rank", "Hrank") is None
assert vocabulary.violation_reason("spend", "Svend") is None
def test_protection_allows_possessive_correction_toward_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Pawpaw's exhausted.", "Popov's exhausted.") is None
def test_protection_blocks_noncanonical_introduced_protected_term():
vocabulary = _vocabulary()
assert (
vocabulary.violation_reason("gestures", "jesters")
== "correction changes protected glossary term capitalization"
)
assert (
vocabulary.violation_reason("rank", "hrank")
== "correction changes protected glossary term capitalization"
)
assert (
vocabulary.violation_reason("spend", "svend")
== "correction changes protected glossary term capitalization"
)
def test_protection_blocks_changed_noncanonical_variant():
vocabulary = _vocabulary()
assert (
vocabulary.violation_reason("jesters advance.", "JESTERS advance.")
== "correction changes protected glossary term capitalization"
)
def test_protection_allows_inferred_name_plural():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Godfrey's", "Godfreys") is None
def test_protection_allows_inferred_alias_plural():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("gesture", "Jesters") is None
def test_protection_allows_explicit_plural():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Mox"
plural: "Moxen"
category: faction
summary: "The Mox are a faction."
"""
)
vocabulary = ProtectedVocabulary.from_glossary(glossary)
assert vocabulary.violation_reason("Mox's", "Moxen") is None
assert (
vocabulary.violation_reason("Mox's", "moxen")
== "correction changes protected glossary term capitalization"
)
def test_protection_allows_punctuation_around_protected_term():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Popov, moves.", "Popov. Moves.") is None
def test_protection_allows_quote_wrapping_sentence_with_unchanged_lowercase_protected_term():
vocabulary = _vocabulary()
before = (
"When you say that, Popov will say, when I was in that room with the jesters, "
"I just knew that Godfrey and Lyra came directly from Loviator herself."
)
after = (
'When you say that, Popov will say, "When I was in that room with the jesters, '
'I just knew that Godfrey and Lyra came directly from Loviator herself."'
)
assert vocabulary.violation_reason(before, after) is None
def test_protection_does_not_match_terms_inside_larger_words():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("The shrank spell worked.", "The shrank spell works.") is None
def test_protection_applies_to_aliases():
vocabulary = _vocabulary()
assert vocabulary.violation_reason("Greenfield waits.", "greenfield waits.") is not None
def test_protection_blocks_removing_preexisting_protected_occurrence():
vocabulary = _vocabulary()
assert (
vocabulary.violation_reason("Jesters flank the Jesters.", "Jesters flank the gestures.")
== "correction changes protected glossary term usage"
)

336
tests/audita_prototype/test_semantic_validation.py
View File

@@ -1,336 +0,0 @@
import pytest
from audita_prototype.errors import AuditaLLMError
from audita_prototype.protection import ProtectedVocabulary
from audita_prototype.schemas import (
CorrectionCandidate,
GrammarSpokenFormValidationDecision,
GrammarSpokenFormValidationSet,
GrammarValidationDecision,
GrammarValidationSet,
parse_glossary_yaml,
parse_transcript_json,
)
from audita_prototype.semantic_validation import (
filter_with_meaning_preserving_validations,
filter_with_spoken_form_validations,
keep_corrections_with_indexes,
select_grammar_validation_candidates,
)
def _transcript():
return parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "he became visible and then gestures arrived"}
]
"""
)
def _vocabulary():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Jesters"
category: faction
summary: "The Jesters are a faction."
"""
)
return ProtectedVocabulary.from_glossary(glossary)
def test_protected_vocabulary_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="gestures",
corrected_text="Jesters",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_capitalization_only_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="he",
corrected_text="He",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_punctuation_only_correction_bypasses_validation():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="visible.",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert candidates == []
assert bypassed_count == 1
def test_meaning_sensitive_substitution_requires_validation():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, bypassed_count = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
assert len(candidates) == 1
assert candidates[0].correction_index == 0
assert candidates[0].original_segment_text == "he became visible and then gestures arrived"
assert candidates[0].corrected_segment_text == "he became invisible and then gestures arrived"
assert bypassed_count == 0
def test_meaning_preserving_validation_approves_without_rescue():
correction = CorrectionCandidate(
id=1,
original_text="bind",
corrected_text="mind",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Keep in bind."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="This preserves the intended meaning.",
)
]
),
confidence_threshold=0.8,
)
assert result.approved_correction_indexes == [0]
assert result.rescue_candidates == []
assert result.approved_count == 1
assert result.rejected_count == 0
def test_spoken_form_validation_can_rescue_homophone_fix():
correction = CorrectionCandidate(
id=1,
original_text="dam",
corrected_text="damn",
confidence=0.95,
)
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "ChatGPT still can't really do that with a dam."}
]
"""
)
candidates, _ = select_grammar_validation_candidates(
transcript,
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="Written meaning changes from a barrier to a curse word.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="The surrounding phrase strongly supports the intended spoken phrase with a curse word.",
)
]
),
confidence_threshold=0.8,
)
kept = keep_corrections_with_indexes([correction], spoken_form_result.approved_correction_indexes)
assert meaning_result.approved_correction_indexes == []
assert kept == [correction]
assert spoken_form_result.skipped == []
def test_spoken_form_validation_rejects_non_homophone_semantic_change():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, _ = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_result = filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(
validations=[
GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=False,
confidence=0.99,
reason="This reverses visible to invisible.",
)
]
),
confidence_threshold=0.8,
)
spoken_form_result = filter_with_spoken_form_validations(
meaning_result.rescue_candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=False,
confidence=0.99,
reason="This is a semantic reversal, not a likely spoken-form transcription error.",
)
]
),
confidence_threshold=0.8,
)
assert spoken_form_result.approved_correction_indexes == []
assert spoken_form_result.rejected_count == 1
assert spoken_form_result.skipped[0].reason == "grammar validation rejected semantic change"
assert spoken_form_result.skipped[0].validation_reason == (
"This is a semantic reversal, not a likely spoken-form transcription error."
)
def test_validation_rejects_duplicate_unknown_and_missing_decisions():
correction = CorrectionCandidate(
id=1,
original_text="visible",
corrected_text="invisible",
confidence=0.95,
)
candidates, _ = select_grammar_validation_candidates(
_transcript(),
[correction],
confidence_threshold=0.8,
replacement_mode="require_unique",
protected_vocabulary=_vocabulary(),
)
meaning_decision = GrammarValidationDecision(
correction_index=0,
is_meaning_preserving=True,
confidence=0.95,
reason="Preserves meaning.",
)
spoken_form_decision = GrammarSpokenFormValidationDecision(
correction_index=0,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Likely spoken-form correction.",
)
with pytest.raises(AuditaLLMError):
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(validations=[meaning_decision, meaning_decision]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_meaning_preserving_validations(
candidates,
GrammarValidationSet(validations=[]),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(
validations=[
GrammarSpokenFormValidationDecision(
correction_index=99,
is_likely_spoken_form_correction=True,
confidence=0.95,
reason="Unknown.",
)
]
),
confidence_threshold=0.8,
)
with pytest.raises(AuditaLLMError):
filter_with_spoken_form_validations(
candidates,
GrammarSpokenFormValidationSet(validations=[spoken_form_decision, spoken_form_decision]),
confidence_threshold=0.8,
)

225
tests/audita_prototype/test_validation.py
View File

@@ -1,225 +0,0 @@
import pytest
from audita_prototype.errors import AuditaValidationError
from audita_prototype.schemas import parse_glossary_yaml, parse_source_transcript_json, parse_transcript_json
def test_valid_transcript_parses():
segments = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Then Lyra."}
]
"""
)
assert len(segments) == 1
assert segments[0].id == 1
assert segments[0].speaker == "Eric"
def test_transcript_rejects_extra_fields():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.25, "text": "Hi", "extra": true}
]
"""
)
def test_transcript_rejects_bad_timestamps():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_missing_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_duplicate_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 1, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_nonsequential_ids():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 3, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
def test_transcript_rejects_zero_or_negative_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 0, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_noninteger_id():
with pytest.raises(AuditaValidationError):
parse_transcript_json(
"""
[
{"id": 1.5, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_transcript_rejects_empty_input():
with pytest.raises(AuditaValidationError):
parse_transcript_json("[]")
def test_source_transcript_accepts_missing_ids():
segments = parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
assert segments[0].id is None
assert segments[0].speaker == "Eric"
def test_source_transcript_accepts_present_nonsequential_ids():
segments = parse_source_transcript_json(
"""
[
{"id": 10, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi"},
{"id": 4, "speaker": "Mike", "start": 1.0, "end": 2.0, "text": "There"}
]
"""
)
assert [segment.id for segment in segments] == [10, 4]
def test_source_transcript_rejects_extra_fields():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Hi", "extra": true}
]
"""
)
def test_source_transcript_rejects_bad_timestamps():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "Eric", "start": 2.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_source_transcript_rejects_empty_values():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json(
"""
[
{"speaker": "", "start": 0.0, "end": 1.0, "text": "Hi"}
]
"""
)
def test_source_transcript_rejects_invalid_json():
with pytest.raises(AuditaValidationError):
parse_source_transcript_json("{")
def test_valid_glossary_parses():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
"""
)
assert glossary.glossary[0].name == "Lyra"
assert glossary.glossary[0].plural is None
def test_glossary_accepts_optional_plural():
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Godfrey"
plural: "Godfreys"
category: npc
summary: "Godfrey is an NPC."
"""
)
assert glossary.glossary[0].plural == "Godfreys"
def test_glossary_rejects_empty_plural():
with pytest.raises(AuditaValidationError):
parse_glossary_yaml(
"""
glossary:
- name: "Godfrey"
plural: ""
category: npc
summary: "Godfrey is an NPC."
"""
)
def test_glossary_rejects_empty_entries():
with pytest.raises(AuditaValidationError):
parse_glossary_yaml("glossary: []")
def test_glossary_rejects_extra_fields():
with pytest.raises(AuditaValidationError):
parse_glossary_yaml(
"""
glossary:
- name: "Lyra"
category: npc
summary: "Lyra is a hostile NPC."
extra: "nope"
"""
)