eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add transcript description prompt context

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 12:13:32 +00:00
parent de99467ede
commit ebbd2c8a63
27 changed files with 542 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
README.md
View File

@@ -75,6 +75,17 @@ audita process transcript.json \
--report-json report.json
```
Optional transcript background context:
```sh
audita process transcript.json \
--glossary glossary.yaml \
--transcript-description "Brief context that may help resolve ambiguous terms." \
--output corrected.json
```
The transcript description is background context only and does not override transcript content.
Write transcript JSON to stdout (no `--output`):
```sh
@@ -113,6 +124,17 @@ Precedence:
- `AUDITA_MODULES` (CSV)
- CLI: `--modules`
### Transcript Description
CLI:
- `--transcript-description`
Behavior:
- optional background context for proposal and LLM-validator prompts;
- trimmed and length-limited by CLI validation;
- does not override transcript content;
- no `AUDITA_*` environment variable is currently defined for this setting.
### Primary LLM
Environment:

20
docs/architecture.md
View File

@@ -221,6 +221,7 @@ Implemented config surfaces include:
- module list
- primary and validation LLM settings
- total/proposal/validation LLM concurrency controls
- transcript description context (`--transcript-description`)
- section token controls and target sections
- confidence thresholds
- normalization controls
@@ -229,6 +230,25 @@ Implemented config surfaces include:
Current caveat:
- LLM/module-related settings are active for default and explicit module-run paths.
Transcript description behavior:
- `--transcript-description` is a process-flag input for optional user-supplied background context.
- runtime config stores this value in `Config.TranscriptDescription` after CLI trimming and length validation.
- default value is empty; empty values produce no prompt context section.
- this value is intentionally non-secret and appears in effective config and invocation metadata artifacts.
## Implemented transcript description prompt context
Transcript description context is wired through production prompt paths:
- proposal prompts for `glossary`, `homophones`, `spoken_word`, and `grammar`;
- LLM-backed validator prompts for spoken-form plausibility, meaning reversal, editorial review, grammar review, and spoken-word review.
Prompt guardrail semantics are consistent across modules and validators:
- transcript description is labeled as "background context only";
- it may help interpret ambiguous terms;
- it must not override transcript content;
- the model must not invent corrections, facts, names, events, motivations, or speaker intent from this description.
Generated transcript descriptions remain deferred and are not implemented in the current runtime.
## Implemented structured LLM infrastructure
`internal/framework/contracts` now defines a typed structured-completion contract:
- `StructuredLLMClient.CompleteStructured(ctx, req, out)`

7
docs/roadmap.md
View File

@@ -691,6 +691,13 @@ The description should be included in every proposal and validator prompt as bac
The prompt should clearly state that the description may help interpret ambiguous terms but must not override the transcript.
Implementation status (2026-05-13):
- implemented via `audita process --transcript-description <text>`;
- stored in runtime config as transcript description context and propagated through proposal-generation and LLM-validator prompt builders;
- prompt text explicitly marks this context as background-only and non-authoritative;
- prompt text explicitly forbids inventing corrections, facts, names, events, motivations, or speaker intent from the description;
- empty descriptions do not add blank context sections.
Generated transcript descriptions should remain opt-in or deferred. If implemented before 1.0, they should be:
- explicitly requested;

17
internal/cli/run.go
View File

@@ -74,12 +74,13 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
}
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
Modules: append([]string(nil), inv.Config.Modules...),
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
TranscriptDescription: inv.Config.TranscriptDescription,
Modules: append([]string(nil), inv.Config.Modules...),
}); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
}
@@ -435,6 +436,8 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
case "normalize-max-segment-tokens":
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
case "transcript-description":
overrides.TranscriptDescription = pFlags.transcriptDescription
case "work-dir":
overrides.WorkDir = pFlags.workDir
case "work-dir-retention":
@@ -710,6 +713,7 @@ type processFlags struct {
normalizeEllipsisGap *float64
normalizeMaxSegmentDuration *float64
normalizeMaxSegmentTokens *int
transcriptDescription *string
workDir *string
workDirRetention *string
}
@@ -769,6 +773,7 @@ func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, proc
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
}

231
internal/cli/run_test.go
View File

@@ -15,6 +15,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -83,6 +84,7 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
"--normalize-ellipsis-gap",
"--normalize-max-segment-duration",
"--normalize-max-segment-tokens",
"--transcript-description",
"--work-dir",
"--work-dir-retention",
} {
@@ -249,6 +251,105 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
}
}
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "speaker background context" {
t.Fatalf("expected trimmed transcript description, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--transcript-description", " speaker background context ",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessRejectsOverlyLongTranscriptDescription(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
tooLong := strings.Repeat("a", config.DefaultTranscriptDescriptionMaxChars+1)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--transcript-description", tooLong,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for overly long transcript description")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "transcript description must be 500 characters or fewer") {
t.Fatalf("expected transcript description length validation error, got %q", stderr.String())
}
}
func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -1091,6 +1192,35 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
}
}
type capturePromptStructuredLLMClient struct {
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
requests []contracts.StructuredCompletionRequest
}
func (c *capturePromptStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.requests = append(c.requests, req)
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
if len(c.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = c.proposalResponses[0]
c.proposalResponses = c.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *validators.LLMValidationResponse:
if len(c.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = c.validationResponses[0]
c.validationResponses = c.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
@@ -1466,6 +1596,75 @@ func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *test
}
}
func TestRunProcessTranscriptDescriptionReachesProposalAndValidatorPrompts(t *testing.T) {
proposalClient := &capturePromptStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
}},
},
}
validationClient := &capturePromptStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
description := "Hearing transcript where speakers reference proper nouns."
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--transcript-description", description,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if len(proposalClient.requests) == 0 {
t.Fatalf("expected proposal LLM requests")
}
if len(validationClient.requests) == 0 {
t.Fatalf("expected validation LLM requests")
}
proposalPrompt := combinedPrompt(proposalClient.requests[0].Messages)
validatorPrompt := combinedPrompt(validationClient.requests[0].Messages)
for _, prompt := range []string{proposalPrompt, validatorPrompt} {
for _, want := range []string{
"Transcript description (background context only):",
description,
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(prompt, want) {
t.Fatalf("expected prompt to contain %q, got: %q", want, prompt)
}
}
}
}
func combinedPrompt(messages []contracts.LLMMessage) string {
parts := make([]string, 0, len(messages))
for _, m := range messages {
parts = append(parts, m.Content)
}
return strings.Join(parts, "\n")
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
@@ -2957,6 +3156,8 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--transcript-description",
" scene takes place during a council hearing ",
"--total-llm-concurrency",
"3",
"--proposal-llm-concurrency",
@@ -3006,9 +3207,10 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
}
var effectiveConfig struct {
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TranscriptDescription string `json:"TranscriptDescription"`
}
if err := json.Unmarshal(configBytes, &effectiveConfig); err != nil {
t.Fatalf("failed to parse effective config metadata: %v", err)
@@ -3022,16 +3224,20 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if effectiveConfig.ValidationLLMConcurrency == nil || *effectiveConfig.ValidationLLMConcurrency != 1 {
t.Fatalf("expected validation_llm_concurrency=1 in effective config, got %#v", effectiveConfig.ValidationLLMConcurrency)
}
if effectiveConfig.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("expected transcript description in effective config, got %q", effectiveConfig.TranscriptDescription)
}
var invocation struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
TranscriptDescription string `json:"transcript_description"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
}
if err := json.Unmarshal(invocationBytes, &invocation); err != nil {
t.Fatalf("failed to parse invocation metadata: %v", err)
@@ -3051,6 +3257,9 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if invocation.ReportJSONPath != reportPath {
t.Fatalf("unexpected report_json_path: %q", invocation.ReportJSONPath)
}
if invocation.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("unexpected transcript_description: %q", invocation.TranscriptDescription)
}
if len(invocation.Modules) == 0 {
t.Fatalf("expected non-empty modules list in invocation metadata")
}

34
internal/core/config/config.go
View File

@@ -14,22 +14,23 @@ const (
)
const (
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultTranscriptDescriptionMaxChars = 500
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
)
type Config struct {
@@ -45,6 +46,7 @@ type Config struct {
TargetSections *int
Thresholds ConfidenceThresholds
Normalization NormalizationConfig
TranscriptDescription string
WorkDir string
WorkDirRetention WorkDirRetention
}

26
internal/core/config/config_test.go
View File

@@ -51,6 +51,9 @@ func TestDefaultConfigValues(t *testing.T) {
if cfg.WorkDir != DefaultWorkDir {
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
}
if cfg.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
}
if cfg.WorkDirRetention != DefaultWorkDirRetention {
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
}
@@ -214,6 +217,29 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
}
}
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
cfg := Default()
description := " background context about speakers "
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TranscriptDescription != "background context about speakers" {
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
}
}
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
cfg := Default()
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
err := cfg.Validate()
if err == nil {
t.Fatalf("expected transcript description length validation error")
}
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
t.Fatalf("unexpected validation error: %v", err)
}
}
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
cfg := Default()
aliasConcurrency := 6

9
internal/core/config/flags.go
View File

@@ -1,6 +1,9 @@
package config
import "fmt"
import (
"fmt"
"strings"
)
type CLIOverrides struct {
ModulesCSV *string
@@ -30,6 +33,7 @@ type CLIOverrides struct {
NormalizeEllipsisGap *float64
NormalizeMaxSegmentDuration *float64
NormalizeMaxSegmentTokens *int
TranscriptDescription *string
WorkDir *string
WorkDirRetention *string
}
@@ -135,6 +139,9 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.NormalizeMaxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
}
if overrides.TranscriptDescription != nil {
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
}
if overrides.WorkDir != nil {
c.WorkDir = *overrides.WorkDir
}

3
internal/core/config/validation.go
View File

@@ -88,6 +88,9 @@ func (c Config) Validate() error {
if c.Normalization.MaxSegmentTokens <= 0 {
issues = append(issues, "normalize max segment tokens must be greater than zero")
}
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
}
if strings.TrimSpace(c.WorkDir) == "" {
issues = append(issues, "work dir must not be empty")

17
internal/core/diagnostics/run_dir.go
View File

@@ -48,14 +48,15 @@ func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
// InvocationMetadata captures non-secret invocation details for diagnostics.
type InvocationMetadata struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
TranscriptDescription string `json:"transcript_description,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}
// NewRunDirectory creates a new run directory under the configured work dir

21
internal/framework/promptcontext/transcript_description.go Normal file
View File

@@ -0,0 +1,21 @@
package promptcontext
import (
"fmt"
"strings"
)
// TranscriptDescriptionBlock returns standardized background-only context
// guidance for prompt builders when a user-supplied description is present.
func TranscriptDescriptionBlock(transcriptDescription string) string {
description := strings.TrimSpace(transcriptDescription)
if description == "" {
return ""
}
return "Transcript description (background context only):\n" +
fmt.Sprintf("%s\n\n", description) +
"Use this description only as optional background to interpret ambiguous terms. " +
"It must not override the transcript content. " +
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.\n\n"
}

29
internal/framework/promptcontext/transcript_description_test.go Normal file
View File

@@ -0,0 +1,29 @@
package promptcontext
import (
"strings"
"testing"
)
func TestTranscriptDescriptionBlockEmpty(t *testing.T) {
if got := TranscriptDescriptionBlock(""); got != "" {
t.Fatalf("expected empty block for empty description, got %q", got)
}
if got := TranscriptDescriptionBlock(" "); got != "" {
t.Fatalf("expected empty block for whitespace description, got %q", got)
}
}
func TestTranscriptDescriptionBlockIncludesGuardrails(t *testing.T) {
got := TranscriptDescriptionBlock("Council hearing with multiple speakers.")
for _, want := range []string{
"Transcript description (background context only):",
"Council hearing with multiple speakers.",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(got, want) {
t.Fatalf("expected block to contain %q", want)
}
}
}

19
internal/framework/validators/llm_prompt_builders.go
View File

@@ -3,9 +3,11 @@ package validators
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -22,11 +24,12 @@ func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem)
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -43,11 +46,12 @@ func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMM
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildEditorialMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -69,16 +73,17 @@ func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) {

8
internal/framework/validators/llm_validators.go
View File

@@ -13,7 +13,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
type LLMBackedValidator struct {
name string
@@ -84,7 +84,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
messages, err := v.promptBuilder(batch.Items)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := v.promptBuilder(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
}

25
internal/framework/validators/llm_validators_test.go
View File

@@ -167,7 +167,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
tests := []struct {
name string
build func([]LLMValidationItem) ([]LLMMessage, error)
build func([]LLMValidationItem, string) ([]LLMMessage, error)
mustHas []string
}{
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
@@ -179,7 +179,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msgs, err := tt.build(payload)
msgs, err := tt.build(payload, "Discussion among party members in a dungeon.")
if err != nil {
t.Fatalf("build err: %v", err)
}
@@ -192,10 +192,31 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
for _, needle := range []string{
"Transcript description (background context only):",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
})
}
}
func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}}
msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ")
if err != nil {
t.Fatalf("build err: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},

6
internal/modules/glossary/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

20
internal/modules/glossary/module_test.go
View File

@@ -61,7 +61,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -91,6 +91,24 @@ func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Campaign scene in a crowded harbor.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, want := range []string{
"Transcript description (background context only):",
"Campaign scene in a crowded harbor.",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, want) {
t.Fatalf("expected prompt to contain %q", want)
}
}
}
func TestGlossaryModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/glossary/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -22,7 +23,7 @@ type promptTranscriptSection struct {
Segments []promptSegment `json:"segments"`
}
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -71,6 +72,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Glossary:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/grammar/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

16
internal/modules/grammar/module_test.go
View File

@@ -71,7 +71,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -97,6 +97,20 @@ func TestBuildProposalMessagesConstraints(t *testing.T) {
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Courtroom exchange with formal titles.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "must not override the transcript content") {
t.Fatalf("expected no-override guidance in prompt")
}
}
func TestGrammarModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/grammar/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to punctuation/capitalization/
// spacing cleanup with strict meaning guards.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -76,6 +77,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/homophones/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

27
internal/modules/homophones/module_test.go
View File

@@ -62,7 +62,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -100,6 +100,31 @@ func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testi
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "must not override the transcript content") {
t.Fatalf("expected no-override guidance in prompt")
}
}
func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/homophones/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to conservative homophone and
// mistranscription updates.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -74,6 +75,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/spoken_word/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

16
internal/modules/spoken_word/module_test.go
View File

@@ -62,7 +62,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -99,6 +99,20 @@ func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T)
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Participants are discussing raid logistics.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.") {
t.Fatalf("expected anti-invention guidance in prompt")
}
}
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/spoken_word/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to conservative dysfluency
// cleanup with strict semantic preservation.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -77,6 +78,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{