eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 17 parity fixture suite

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-12 12:50:05 +00:00
parent 7ccadc6bd6
commit 185f7ca2b6
41 changed files with 798 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/cli/testdata/parity/application-skip-ambiguous.case.json vendored Normal file
View File

@@ -0,0 +1,22 @@
{
"name": "ambiguous_match_rejected_before_application",
"transcript_file": "application-skip-ambiguous.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
"validation_responses_file": "application-skip-ambiguous.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["ambiguous_original_text"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
]

3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
]

4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json vendored Normal file
View File

@@ -0,0 +1,4 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

28
internal/cli/testdata/parity/default-full-pipeline.case.json vendored Normal file
View File

@@ -0,0 +1,28 @@
{
"name": "default_full_pipeline_shape_and_reports",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"proposal_responses_file": "default-full-pipeline.proposals.json",
"validation_responses_file": "default-full-pipeline.validations.json",
"env": {
"AUDITA_LLM_API_KEY": "phase17-secret",
"AUDITA_VALIDATION_LLM_API_KEY": "phase17-secret"
},
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 3,
"total_skipped_changes": 3,
"secret_markers": ["phase17-secret"],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
]
}
}

9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,9 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml vendored Normal file
View File

@@ -0,0 +1,7 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

28
internal/cli/testdata/parity/default-full-pipeline.proposals.json vendored Normal file
View File

@@ -0,0 +1,28 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

3
internal/cli/testdata/parity/default-full-pipeline.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

12
internal/cli/testdata/parity/default-full-pipeline.validations.json vendored Normal file
View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "phase17-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

27
internal/cli/testdata/parity/default-handoff.case.json vendored Normal file
View File

@@ -0,0 +1,27 @@
{
"name": "default_pipeline_handoff_and_module_order",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "default-handoff.proposals.json",
"validation_responses_file": "default-handoff.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-handoff.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 5,
"total_skipped_changes": 0,
"module_applied_counts": [1, 1, 1, 1, 1],
"module_rejected_counts": [0, 0, 0, 0, 0],
"module_skip_counts": [0, 0, 0, 0, 0],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
],
"min_response_payload_artifacts": 15
}
}

3
internal/cli/testdata/parity/default-handoff.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
]

6
internal/cli/testdata/parity/default-handoff.glossary.yaml vendored Normal file
View File

@@ -0,0 +1,6 @@
glossary:
- name: Jesters
aliases: [jester]
plural: jesters
category: faction
summary: A protected in-world faction term.

7
internal/cli/testdata/parity/default-handoff.proposals.json vendored Normal file
View File

@@ -0,0 +1,7 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/default-handoff.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
]

12
internal/cli/testdata/parity/default-handoff.validations.json vendored Normal file
View File

@@ -0,0 +1,12 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"name": "deterministic_validator_low_confidence",
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["low_confidence"],
"expected_proposal_calls": ["grammar:proposal"]
}
}

3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json vendored Normal file
View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
]
}
]

3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

12
internal/cli/testdata/parity/glossary-schema-error.case.json vendored Normal file
View File

@@ -0,0 +1,12 @@
{
"name": "glossary_schema_handling",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "glossary-schema-error.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "glossary_schema",
"stderr_contains": "glossary_schema",
"require_error_log": true
}
}

2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml vendored Normal file
View File

@@ -0,0 +1,2 @@
glossary:
- name: MissingCategoryAndSummary

19
internal/cli/testdata/parity/llm-validator-rejection.case.json vendored Normal file
View File

@@ -0,0 +1,19 @@
{
"name": "llm_validator_decision_handling",
"transcript_file": "llm-validator-rejection.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "llm-validator-rejection.proposals.json",
"validation_responses_file": "llm-validator-rejection.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:grammar_only_guard:batch-0000"]
}
}

3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json vendored Normal file
View File

@@ -0,0 +1,7 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

7
internal/cli/testdata/parity/llm-validator-rejection.validations.json vendored Normal file
View File

@@ -0,0 +1,7 @@
[
{
"validations": [
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
]
}
]

23
internal/cli/testdata/parity/mid-pipeline-failure.case.json vendored Normal file
View File

@@ -0,0 +1,23 @@
{
"name": "mid_pipeline_failure_partial_progress",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
"validation_responses_file": "mid-pipeline-failure.validations.json",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "runner_execution",
"stderr_contains": "runner_execution",
"module_instances": ["glossary_1", "homophones", "glossary_2"],
"module_count": 3,
"total_applied_changes": 2,
"total_skipped_changes": 0,
"failed_module_instance": "glossary_2",
"module_applied_counts": [1, 1, 0],
"module_rejected_counts": [0, 0, 0],
"module_skip_counts": [0, 0, 0],
"require_error_log": true,
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
}
}

4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json vendored Normal file
View File

@@ -0,0 +1,4 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json vendored Normal file
View File

@@ -0,0 +1,6 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

21
internal/cli/testdata/parity/protected-term-rejection.case.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"name": "protected_glossary_term_behavior",
"transcript_file": "protected-term-rejection.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "protected-term-rejection.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["protected_glossary_term"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
]

3
internal/cli/testdata/parity/protected-term-rejection.proposals.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/protected-term-rejection.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
]

12
internal/cli/testdata/parity/transcript-schema-error.case.json vendored Normal file
View File

@@ -0,0 +1,12 @@
{
"name": "transcript_schema_handling",
"transcript_file": "transcript-schema-error.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "transcript_schema",
"stderr_contains": "transcript_schema",
"require_error_log": true
}
}

3
internal/cli/testdata/parity/transcript-schema-error.transcript.json vendored Normal file
View File

@@ -0,0 +1,3 @@
[
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
]