eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Migrate examples and guidance and remove legacy test harness

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-24 02:37:20 +00:00
parent 9561e80efb
commit 80804a038f
102 changed files with 328 additions and 8371 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/cli/testdata/parity/application-skip-ambiguous.case.json vendored
View File

@@ -1,22 +0,0 @@
{
"name": "ambiguous_match_rejected_before_application",
"transcript_file": "application-skip-ambiguous.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "application-skip-ambiguous.proposals.json",
"validation_responses_file": "application-skip-ambiguous.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "application-skip-ambiguous.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["ambiguous_original_text"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

3
internal/cli/testdata/parity/application-skip-ambiguous.expected-transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"the site near another site"}
]

3
internal/cli/testdata/parity/application-skip-ambiguous.proposals.json vendored
View File

@@ -1,3 +0,0 @@
[
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/application-skip-ambiguous.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"the site near another site"}
]

4
internal/cli/testdata/parity/application-skip-ambiguous.validations.json vendored
View File

@@ -1,4 +0,0 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

28
internal/cli/testdata/parity/default-full-pipeline.case.json vendored
View File

@@ -1,28 +0,0 @@
{
"name": "default_full_pipeline_shape_and_reports",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"proposal_responses_file": "default-full-pipeline.proposals.json",
"validation_responses_file": "default-full-pipeline.validations.json",
"env": {
"AUDITA_LLM_API_KEY": "parity-secret",
"AUDITA_VALIDATION_LLM_API_KEY": "parity-secret"
},
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-full-pipeline.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 3,
"total_skipped_changes": 3,
"secret_markers": ["parity-secret"],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
]
}
}

9
internal/cli/testdata/parity/default-full-pipeline.expected-transcript.json vendored
View File

@@ -1,9 +0,0 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

7
internal/cli/testdata/parity/default-full-pipeline.glossary.yaml vendored
View File

@@ -1,7 +0,0 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

28
internal/cli/testdata/parity/default-full-pipeline.proposals.json vendored
View File

@@ -1,28 +0,0 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

3
internal/cli/testdata/parity/default-full-pipeline.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

12
internal/cli/testdata/parity/default-full-pipeline.validations.json vendored
View File

@@ -1,12 +0,0 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "parity-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

27
internal/cli/testdata/parity/default-handoff.case.json vendored
View File

@@ -1,27 +0,0 @@
{
"name": "default_pipeline_handoff_and_module_order",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "default-handoff.proposals.json",
"validation_responses_file": "default-handoff.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "default-handoff.expected-transcript.json",
"module_instances": ["glossary_1", "homophones", "glossary_2", "spoken_word", "grammar"],
"module_count": 5,
"total_applied_changes": 5,
"total_skipped_changes": 0,
"module_applied_counts": [1, 1, 1, 1, 1],
"module_rejected_counts": [0, 0, 0, 0, 0],
"module_skip_counts": [0, 0, 0, 0, 0],
"expected_proposal_calls": [
"glossary_1:proposal",
"homophones:proposal",
"glossary_2:proposal",
"spoken_word:proposal",
"grammar:proposal"
],
"min_response_payload_artifacts": 15
}
}

3
internal/cli/testdata/parity/default-handoff.expected-transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"Hello, there were Jesters at the Sight um"}
]

6
internal/cli/testdata/parity/default-handoff.glossary.yaml vendored
View File

@@ -1,6 +0,0 @@
glossary:
- name: Jesters
aliases: [jester]
plural: jesters
category: faction
summary: A protected in-world faction term.

7
internal/cli/testdata/parity/default-handoff.proposals.json vendored
View File

@@ -1,7 +0,0 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "sight", "corrected_text": "Sight", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "um um", "corrected_text": "um", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/default-handoff.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures at the site um um"}
]

12
internal/cli/testdata/parity/default-handoff.validations.json vendored
View File

@@ -1,12 +0,0 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

17
internal/cli/testdata/parity/deterministic-validator-low-confidence.case.json vendored
View File

@@ -1,17 +0,0 @@
{
"name": "deterministic_validator_low_confidence",
"transcript_file": "deterministic-validator-low-confidence.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "deterministic-validator-low-confidence.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "deterministic-validator-low-confidence.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["low_confidence"],
"expected_proposal_calls": ["grammar:proposal"]
}
}

3
internal/cli/testdata/parity/deterministic-validator-low-confidence.expected-transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

7
internal/cli/testdata/parity/deterministic-validator-low-confidence.proposals.json vendored
View File

@@ -1,7 +0,0 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.1}
]
}
]

3
internal/cli/testdata/parity/deterministic-validator-low-confidence.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

12
internal/cli/testdata/parity/glossary-schema-error.case.json vendored
View File

@@ -1,12 +0,0 @@
{
"name": "glossary_schema_handling",
"transcript_file": "default-full-pipeline.transcript.json",
"glossary_file": "glossary-schema-error.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "glossary_schema",
"stderr_contains": "glossary_schema",
"require_error_log": true
}
}

2
internal/cli/testdata/parity/glossary-schema-error.glossary.yaml vendored
View File

@@ -1,2 +0,0 @@
glossary:
- name: MissingCategoryAndSummary

19
internal/cli/testdata/parity/llm-validator-rejection.case.json vendored
View File

@@ -1,19 +0,0 @@
{
"name": "llm_validator_decision_handling",
"transcript_file": "llm-validator-rejection.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"modules_csv": "grammar",
"proposal_responses_file": "llm-validator-rejection.proposals.json",
"validation_responses_file": "llm-validator-rejection.validations.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "llm-validator-rejection.expected-transcript.json",
"module_instances": ["grammar"],
"module_count": 1,
"total_skipped_changes": 1,
"validator_rejected_reason_codes": ["llm_rejected"],
"expected_proposal_calls": ["grammar:proposal"],
"expected_validation_calls": ["grammar:section-0000:editorial_review:batch-0000"]
}
}

3
internal/cli/testdata/parity/llm-validator-rejection.expected-transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"hello , world"}
]

7
internal/cli/testdata/parity/llm-validator-rejection.proposals.json vendored
View File

@@ -1,7 +0,0 @@
[
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

3
internal/cli/testdata/parity/llm-validator-rejection.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , world"}
]

7
internal/cli/testdata/parity/llm-validator-rejection.validations.json vendored
View File

@@ -1,7 +0,0 @@
[
{
"validations": [
{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject stylistic overreach"}
]
}
]

23
internal/cli/testdata/parity/mid-pipeline-failure.case.json vendored
View File

@@ -1,23 +0,0 @@
{
"name": "mid_pipeline_failure_partial_progress",
"transcript_file": "default-handoff.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"proposal_responses_file": "mid-pipeline-failure.proposals.json",
"validation_responses_file": "mid-pipeline-failure.validations.json",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "runner_execution",
"stderr_contains": "runner_execution",
"module_instances": ["glossary_1", "homophones", "glossary_2"],
"module_count": 3,
"total_applied_changes": 2,
"total_skipped_changes": 0,
"failed_module_instance": "glossary_2",
"module_applied_counts": [1, 1, 0],
"module_rejected_counts": [0, 0, 0],
"module_skip_counts": [0, 0, 0],
"require_error_log": true,
"expected_proposal_calls": ["glossary_1:proposal", "homophones:proposal", "glossary_2:proposal"]
}
}

4
internal/cli/testdata/parity/mid-pipeline-failure.proposals.json vendored
View File

@@ -1,4 +0,0 @@
[
{"corrections": [{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}]},
{"corrections": [{"id": 1, "original_text": "site", "corrected_text": "sight", "confidence": 0.99}]}
]

6
internal/cli/testdata/parity/mid-pipeline-failure.validations.json vendored
View File

@@ -1,6 +0,0 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]

21
internal/cli/testdata/parity/protected-term-rejection.case.json vendored
View File

@@ -1,21 +0,0 @@
{
"name": "protected_glossary_term_behavior",
"transcript_file": "protected-term-rejection.transcript.json",
"glossary_file": "default-handoff.glossary.yaml",
"modules_csv": "homophones",
"proposal_responses_file": "protected-term-rejection.proposals.json",
"expect": {
"exit_code": 0,
"status": "success",
"output_transcript_file": "protected-term-rejection.expected-transcript.json",
"module_instances": ["homophones"],
"module_count": 1,
"total_applied_changes": 0,
"total_skipped_changes": 1,
"module_applied_counts": [0],
"module_rejected_counts": [1],
"module_skip_counts": [0],
"validator_rejected_reason_codes": ["protected_glossary_term"],
"expected_proposal_calls": ["homophones:proposal"]
}
}

3
internal/cli/testdata/parity/protected-term-rejection.expected-transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0,"end":1,"text":"The Jesters entered the hall."}
]

3
internal/cli/testdata/parity/protected-term-rejection.proposals.json vendored
View File

@@ -1,3 +0,0 @@
[
{"corrections": [{"id": 1, "original_text": "Jesters", "corrected_text": "Gestures", "confidence": 0.99}]}
]

3
internal/cli/testdata/parity/protected-term-rejection.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"The Jesters entered the hall."}
]

12
internal/cli/testdata/parity/transcript-schema-error.case.json vendored
View File

@@ -1,12 +0,0 @@
{
"name": "transcript_schema_handling",
"transcript_file": "transcript-schema-error.transcript.json",
"glossary_file": "default-full-pipeline.glossary.yaml",
"expect": {
"exit_code": 1,
"status": "failed",
"error_phase": "transcript_schema",
"stderr_contains": "transcript_schema",
"require_error_log": true
}
}

3
internal/cli/testdata/parity/transcript-schema-error.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"","start":0.0,"end":1.0,"text":"bad"}
]

24
internal/cli/testdata/release/default-release.expectations.json vendored
View File

@@ -1,24 +0,0 @@
{
"must_apply_texts": [
"Hello, there were Jesters hmm"
],
"must_not_apply_texts": [
"JESTERX",
"there were gestures"
],
"protected_terms": [
"Jesters"
],
"expected_module_instances": [
"glossary_1",
"homophones",
"glossary_2",
"spoken_word",
"grammar"
],
"minimum_counts": {
"applied": 1,
"rejected": 1,
"skipped": 0
}
}

9
internal/cli/testdata/release/default-release.expected-transcript.json vendored
View File

@@ -1,9 +0,0 @@
[
{
"id": 1,
"speaker": "Alice",
"start": 0,
"end": 1,
"text": "Hello, there were Jesters hmm"
}
]

7
internal/cli/testdata/release/default-release.glossary.yaml vendored
View File

@@ -1,7 +0,0 @@
glossary:
- name: Jesters
aliases:
- jester
plural: jesters
category: faction
summary: A protected in-world faction term.

28
internal/cli/testdata/release/default-release.proposals.json vendored
View File

@@ -1,28 +0,0 @@
[
{
"corrections": [
{"id": 1, "original_text": "gestures", "corrected_text": "Jesters", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "Jesters", "corrected_text": "jesters", "confidence": 0.99},
{"id": 1, "original_text": "Jesters", "corrected_text": "JESTERX", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "jesters", "corrected_text": "JESTERS", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "uh", "corrected_text": "hmm", "confidence": 0.99}
]
},
{
"corrections": [
{"id": 1, "original_text": "hello ,", "corrected_text": "Hello,", "confidence": 0.99}
]
}
]

3
internal/cli/testdata/release/default-release.transcript.json vendored
View File

@@ -1,3 +0,0 @@
[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello , there were gestures uh"}
]

12
internal/cli/testdata/release/default-release.validations.json vendored
View File

@@ -1,12 +0,0 @@
[
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]},
{"validations": [{"correction_index": 0, "approved": false, "confidence": 0.99, "reason": "reject cleanup"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "release-secret"}]},
{"validations": [{"correction_index": 0, "approved": true, "confidence": 0.99, "reason": "ok"}]}
]