eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Added a configuration flag to set the target number of sections, and updated default min and max section token limits

Browse Source
This commit is contained in:
Eric Rakestraw
2026-04-29 20:10:00 -05:00
parent f841f7eb71
commit 4c899122d9
9 changed files with 312 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
tests/test_framework_chunking.py
View File

@@ -1,4 +1,7 @@
import pytest
from audita.core.chunking import TokenEstimatorProtocol, chunk_transcript
from audita.core.errors import AuditaValidationError
from audita.core.schemas import parse_transcript_json
@@ -100,6 +103,93 @@ def test_chunk_transcript_reduces_section_count_when_target_sections_fall_below_
assert sections[0].token_count >= 8
def test_chunk_transcript_exact_target_sections_returns_exact_count_when_feasible():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"},
{"id": 4, "speaker": "A", "start": 3.0, "end": 4.0, "text": "four"}
]
"""
)
sections = chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=2,
estimator=FakeEstimator(),
)
assert len(sections) == 2
assert [segment.segment.id for segment in sections[0].segments] == [1, 2]
assert [segment.segment.id for segment in sections[1].segments] == [3, 4]
def test_chunk_transcript_exact_target_sections_errors_when_target_exceeds_segment_count():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"}
]
"""
)
with pytest.raises(AuditaValidationError, match="Target section count exceeds the number of transcript segments"):
chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=3,
estimator=FakeEstimator(),
)
def test_chunk_transcript_exact_target_sections_errors_when_section_would_exceed_max():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"):
chunk_transcript(
transcript,
max_section_tokens=8,
min_section_tokens=4,
exact_target_section_count=1,
estimator=FakeEstimator(),
)
def test_chunk_transcript_exact_target_sections_errors_when_section_would_fall_below_min():
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "one"},
{"id": 2, "speaker": "A", "start": 1.0, "end": 2.0, "text": "two"},
{"id": 3, "speaker": "A", "start": 2.0, "end": 3.0, "text": "three"}
]
"""
)
with pytest.raises(AuditaValidationError, match="AUDITA_TARGET_SECTIONS cannot produce contiguous transcript sections"):
chunk_transcript(
transcript,
max_section_tokens=12,
min_section_tokens=8,
exact_target_section_count=3,
estimator=FakeEstimator(),
)
def test_chunk_transcript_prompt_payload_includes_categories_when_present():
transcript = parse_transcript_json(
"""

51
tests/test_framework_runner.py
View File

@@ -498,7 +498,7 @@ def test_pipeline_runner_collects_section_proposals_concurrently_and_preserves_s
module = ConcurrentRecordingModule(seen, threading.Barrier(2, timeout=1.0))
monkeypatch.setattr(
"audita.framework.runner.chunk_transcript",
lambda working, max_tokens, min_section_tokens=1, target_section_count=None: sections,
lambda working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None: sections,
)
runner = PipelineRunner()
@@ -517,6 +517,55 @@ def test_pipeline_runner_collects_section_proposals_concurrently_and_preserves_s
assert result.transcript[1].text == "Beta revised."
def test_pipeline_runner_passes_exact_target_sections_to_chunker(tmp_path, monkeypatch):
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "A", "start": 0.0, "end": 1.0, "text": "Alpha."}
]
"""
)
glossary = parse_glossary_yaml(
"""
glossary:
- name: "Alpha"
category: noun
summary: "Alpha."
"""
)
seen_args = {}
sections = [
TranscriptSection(
section_index=0,
start_index=0,
segments=[IndexedSegment(index=0, segment=transcript[0])],
token_count=1,
)
]
module = RecordingModule("noop", [], [], [])
def _fake_chunk_transcript(working, max_tokens, min_section_tokens=1, target_section_count=None, exact_target_section_count=None):
seen_args["target_section_count"] = target_section_count
seen_args["exact_target_section_count"] = exact_target_section_count
return sections
monkeypatch.setattr("audita.framework.runner.chunk_transcript", _fake_chunk_transcript)
runner = PipelineRunner()
runner.run(
transcript=transcript,
glossary=glossary,
module_specs=[ModuleRunSpec(instance_name="noop", module_key="noop", module=module)],
config=AuditaConfig.from_sources(env={}, overrides=ConfigOverrides(target_sections=3)),
run_dir=tmp_path / "run",
)
assert seen_args == {
"target_section_count": None,
"exact_target_section_count": 3,
}
def test_pipeline_runner_reports_first_llm_validator_rejection_in_chain_order(tmp_path):
transcript = parse_transcript_json(
"""

55
tests/test_new_cli.py
View File

@@ -23,6 +23,7 @@ def test_process_help_exposes_framework_flags(capsys):
assert "--llm-api-key" in output
assert "--llm-concurrency" in output
assert "--llm-timeout-seconds" in output
assert "--target-sections" in output
assert "--modules" in output
assert "--model" in output
assert "--base-url" in output
@@ -307,6 +308,60 @@ def test_cli_process_passes_llm_timeout_seconds_override_to_config(monkeypatch,
assert captured["llm_timeout_seconds"] == 900.0
def test_cli_process_passes_target_sections_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(
"""
[
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "Fixed."}
]
"""
)
report = RunReport(
status="success",
config={"model": "m", "base_url": "b"},
normalization={"source_segment_count": 1, "normalized_segment_count": 1, "merge_count": 0},
pipeline=["grammar"],
modules=[],
applied_changes=[],
skipped_corrections=[],
totals={"output_segment_count": 1, "applied_change_count": 0, "skipped_correction_count": 0},
work_dir_retention="auto",
work_dir_retained=False,
work_dir=None,
error=None,
)
result = ProcessResult(
transcript=transcript,
report=report,
run_dir=tmp_path / "run",
work_dir_retained=False,
)
def _fake_from_sources(*, overrides=None):
captured["target_sections"] = overrides.target_sections
return object()
monkeypatch.setattr("audita.cli.AuditaConfig.from_sources", _fake_from_sources)
monkeypatch.setattr("audita.cli.load_transcript", lambda path: [])
monkeypatch.setattr("audita.cli.load_glossary", lambda path: object())
monkeypatch.setattr("audita.cli.process_transcript_result", lambda *args, **kwargs: result)
exit_code = main(
[
"process",
"transcript.json",
"--glossary",
"glossary.yaml",
"--target-sections",
"4",
]
)
assert exit_code == 0
assert captured["target_sections"] == 4
def test_cli_process_passes_min_section_tokens_override_to_config(monkeypatch, tmp_path):
captured = {}
transcript = parse_transcript_json(

28
tests/test_new_config.py
View File

@@ -24,6 +24,7 @@ def test_default_config_allows_missing_api_key():
assert config.api_key is None
assert config.llm_concurrency == DEFAULT_LLM_CONCURRENCY
assert config.llm_timeout_seconds == DEFAULT_LLM_TIMEOUT_SECONDS
assert config.target_sections is None
assert config.max_section_tokens == DEFAULT_MAX_SECTION_TOKENS
assert config.min_section_tokens == DEFAULT_MIN_SECTION_TOKENS
assert config.module_keys == DEFAULT_MODULE_KEYS
@@ -83,6 +84,21 @@ def test_llm_timeout_seconds_env_is_parsed():
assert config.llm_timeout_seconds == 120.5
def test_target_sections_cli_override_takes_precedence():
config = AuditaConfig.from_sources(
env={"AUDITA_TARGET_SECTIONS": "2"},
overrides=ConfigOverrides(target_sections=5),
)
assert config.target_sections == 5
def test_target_sections_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "3"})
assert config.target_sections == 3
def test_min_section_tokens_env_is_parsed():
config = AuditaConfig.from_sources(env={"AUDITA_MIN_SECTION_TOKENS": "3000"})
@@ -153,6 +169,12 @@ def test_report_dict_includes_llm_timeout_seconds():
assert config.to_report_dict()["llm_timeout_seconds"] == 321.0
def test_report_dict_includes_target_sections():
config = AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": "7"})
assert config.to_report_dict()["target_sections"] == 7
def test_threshold_overrides_take_precedence():
config = AuditaConfig.from_sources(
env={
@@ -214,6 +236,12 @@ def test_invalid_llm_timeout_seconds_is_rejected(value):
AuditaConfig.from_sources(env={"AUDITA_LLM_TIMEOUT_SECONDS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_target_sections_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_TARGET_SECTIONS"):
AuditaConfig.from_sources(env={"AUDITA_TARGET_SECTIONS": value})
@pytest.mark.parametrize("value", ["0", "-1", "many"])
def test_invalid_min_section_tokens_is_rejected(value):
with pytest.raises(AuditaConfigError, match="AUDITA_MIN_SECTION_TOKENS"):