200 lines
7.2 KiB
Python
200 lines
7.2 KiB
Python
import json
|
|
|
|
from audita.chunking import chunk_transcript
|
|
from audita.prompts import (
|
|
build_glossary_correction_messages,
|
|
build_grammar_correction_messages,
|
|
build_grammar_validation_messages,
|
|
)
|
|
from audita.schemas import parse_glossary_yaml, parse_transcript_json
|
|
|
|
|
|
def test_prompt_requires_acoustically_plausible_transcription_errors():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
category: faction
|
|
summary: "The Jesters are a local faction."
|
|
- name: "Lyra"
|
|
category: npc
|
|
summary: "Lyra is a hostile NPC."
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
|
|
|
messages = build_glossary_correction_messages(section, glossary)
|
|
prompt_text = "\n".join(message["content"] for message in messages)
|
|
|
|
assert "acoustically plausible" in prompt_text
|
|
assert "phonetically or acoustically similar" in prompt_text
|
|
assert '"gestures" to "Jesters"' in prompt_text
|
|
assert '"Lyra" to "Jesters"' in prompt_text
|
|
assert "should be omitted" in prompt_text
|
|
assert "glossary names and aliases already present in the transcript as protected spellings" in prompt_text
|
|
assert "Do not replace, Anglicize, normalize, lowercase" in prompt_text
|
|
assert "Preserve canonical glossary capitalization" in prompt_text
|
|
assert "exact text span that needs replacement" in prompt_text
|
|
assert "replacement text for that span" in prompt_text
|
|
assert "Do not return corrections where original_text and corrected_text are identical" in prompt_text
|
|
|
|
|
|
def test_prompt_uses_simplified_segment_payload():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Jesters"
|
|
category: faction
|
|
summary: "The Jesters are a local faction."
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
|
|
|
messages = build_glossary_correction_messages(section, glossary)
|
|
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
|
|
prompt_segments = json.loads(transcript_json)
|
|
|
|
assert prompt_segments == [{"id": 1, "original_text": "The gestures are nearby."}]
|
|
assert "speaker" not in prompt_segments[0]
|
|
assert "start" not in prompt_segments[0]
|
|
assert "end" not in prompt_segments[0]
|
|
|
|
|
|
def test_prompts_do_not_include_inferred_plurals():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "The gestures are nearby."}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Godfrey"
|
|
aliases:
|
|
- "Jester"
|
|
category: npc
|
|
summary: "Godfrey is an NPC."
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
|
|
|
glossary_messages = build_glossary_correction_messages(section, glossary)
|
|
glossary_json = glossary_messages[1]["content"].split("Glossary:\n", maxsplit=1)[1].split(
|
|
"\n\nTranscript section:",
|
|
maxsplit=1,
|
|
)[0]
|
|
grammar_messages = build_grammar_correction_messages(section, glossary)
|
|
grammar_json = grammar_messages[1]["content"].split("Protected glossary/context:\n", maxsplit=1)[1].split(
|
|
"\n\nTranscript section:",
|
|
maxsplit=1,
|
|
)[0]
|
|
|
|
for prompt_glossary in (json.loads(glossary_json), json.loads(grammar_json)):
|
|
entry = prompt_glossary["glossary"][0]
|
|
assert "plural" not in entry
|
|
assert "Godfreys" not in json.dumps(prompt_glossary)
|
|
assert "Jesters" not in json.dumps(prompt_glossary)
|
|
|
|
|
|
def test_grammar_prompt_limits_readability_corrections_and_protects_glossary():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Lyra"
|
|
category: npc
|
|
summary: "Lyra is a hostile NPC."
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
|
|
|
messages = build_grammar_correction_messages(section, glossary)
|
|
prompt_text = "\n".join(message["content"] for message in messages)
|
|
|
|
assert "capitalization" in prompt_text
|
|
assert "commas, periods, em dashes, and ellipses" in prompt_text
|
|
assert "homophone fixes" in prompt_text
|
|
assert "spelling fixes" in prompt_text
|
|
assert "Do not paraphrase" in prompt_text
|
|
assert "glossary names and aliases as protected spellings" in prompt_text
|
|
assert "correct clear transcription or spelling errors toward glossary names or aliases" in prompt_text
|
|
assert "Do not autocorrect, Anglicize, replace, normalize, lowercase" in prompt_text
|
|
assert "that already appear correctly in the transcript" in prompt_text
|
|
assert "Preserve canonical glossary capitalization" in prompt_text
|
|
assert "appears exactly once" in prompt_text
|
|
assert "Do not return speaker, start, or end fields" in prompt_text
|
|
|
|
|
|
def test_grammar_prompt_uses_simplified_segment_payload():
|
|
transcript = parse_transcript_json(
|
|
"""
|
|
[
|
|
{"id": 1, "speaker": "Eric", "start": 0.0, "end": 1.0, "text": "then lyra went their"}
|
|
]
|
|
"""
|
|
)
|
|
glossary = parse_glossary_yaml(
|
|
"""
|
|
glossary:
|
|
- name: "Lyra"
|
|
category: npc
|
|
summary: "Lyra is a hostile NPC."
|
|
"""
|
|
)
|
|
section = chunk_transcript(transcript, max_section_tokens=16000)[0]
|
|
|
|
messages = build_grammar_correction_messages(section, glossary)
|
|
transcript_json = messages[1]["content"].split("Transcript section:\n", maxsplit=1)[1]
|
|
prompt_segments = json.loads(transcript_json)
|
|
|
|
assert prompt_segments == [{"id": 1, "original_text": "then lyra went their"}]
|
|
assert "speaker" not in prompt_segments[0]
|
|
assert "start" not in prompt_segments[0]
|
|
assert "end" not in prompt_segments[0]
|
|
|
|
|
|
def test_grammar_validation_prompt_rejects_semantic_changes():
|
|
messages = build_grammar_validation_messages(
|
|
[
|
|
{
|
|
"correction_index": 0,
|
|
"id": 1,
|
|
"original_segment_text": "He became visible.",
|
|
"corrected_segment_text": "He became invisible.",
|
|
"original_text": "visible",
|
|
"corrected_text": "invisible",
|
|
}
|
|
]
|
|
)
|
|
prompt_text = "\n".join(message["content"] for message in messages)
|
|
|
|
assert "preserves meaning" in prompt_text
|
|
assert "became visible" in prompt_text
|
|
assert "became invisible" in prompt_text
|
|
assert "reverses the meaning" in prompt_text
|
|
assert "spelling and homophone fixes only when" in prompt_text
|
|
assert "correction_index" in prompt_text
|
|
assert "is_meaning_preserving" in prompt_text
|