eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add transcript description prompt context

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 12:13:32 +00:00
parent de99467ede
commit ebbd2c8a63
27 changed files with 542 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/cli/run.go
View File

@@ -74,12 +74,13 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
}
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
Modules: append([]string(nil), inv.Config.Modules...),
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
TranscriptDescription: inv.Config.TranscriptDescription,
Modules: append([]string(nil), inv.Config.Modules...),
}); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
}
@@ -435,6 +436,8 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
case "normalize-max-segment-tokens":
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
case "transcript-description":
overrides.TranscriptDescription = pFlags.transcriptDescription
case "work-dir":
overrides.WorkDir = pFlags.workDir
case "work-dir-retention":
@@ -710,6 +713,7 @@ type processFlags struct {
normalizeEllipsisGap *float64
normalizeMaxSegmentDuration *float64
normalizeMaxSegmentTokens *int
transcriptDescription *string
workDir *string
workDirRetention *string
}
@@ -769,6 +773,7 @@ func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, proc
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
}

231
internal/cli/run_test.go
View File

@@ -15,6 +15,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -83,6 +84,7 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
"--normalize-ellipsis-gap",
"--normalize-max-segment-duration",
"--normalize-max-segment-tokens",
"--transcript-description",
"--work-dir",
"--work-dir-retention",
} {
@@ -249,6 +251,105 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
}
}
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "speaker background context" {
t.Fatalf("expected trimmed transcript description, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--transcript-description", " speaker background context ",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessRejectsOverlyLongTranscriptDescription(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
tooLong := strings.Repeat("a", config.DefaultTranscriptDescriptionMaxChars+1)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--transcript-description", tooLong,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for overly long transcript description")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "transcript description must be 500 characters or fewer") {
t.Fatalf("expected transcript description length validation error, got %q", stderr.String())
}
}
func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -1091,6 +1192,35 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
}
}
type capturePromptStructuredLLMClient struct {
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
requests []contracts.StructuredCompletionRequest
}
func (c *capturePromptStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.requests = append(c.requests, req)
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
if len(c.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = c.proposalResponses[0]
c.proposalResponses = c.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *validators.LLMValidationResponse:
if len(c.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = c.validationResponses[0]
c.validationResponses = c.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
@@ -1466,6 +1596,75 @@ func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *test
}
}
func TestRunProcessTranscriptDescriptionReachesProposalAndValidatorPrompts(t *testing.T) {
proposalClient := &capturePromptStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
}},
},
}
validationClient := &capturePromptStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
description := "Hearing transcript where speakers reference proper nouns."
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--transcript-description", description,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if len(proposalClient.requests) == 0 {
t.Fatalf("expected proposal LLM requests")
}
if len(validationClient.requests) == 0 {
t.Fatalf("expected validation LLM requests")
}
proposalPrompt := combinedPrompt(proposalClient.requests[0].Messages)
validatorPrompt := combinedPrompt(validationClient.requests[0].Messages)
for _, prompt := range []string{proposalPrompt, validatorPrompt} {
for _, want := range []string{
"Transcript description (background context only):",
description,
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(prompt, want) {
t.Fatalf("expected prompt to contain %q, got: %q", want, prompt)
}
}
}
}
func combinedPrompt(messages []contracts.LLMMessage) string {
parts := make([]string, 0, len(messages))
for _, m := range messages {
parts = append(parts, m.Content)
}
return strings.Join(parts, "\n")
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
@@ -2957,6 +3156,8 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--transcript-description",
" scene takes place during a council hearing ",
"--total-llm-concurrency",
"3",
"--proposal-llm-concurrency",
@@ -3006,9 +3207,10 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
}
var effectiveConfig struct {
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TranscriptDescription string `json:"TranscriptDescription"`
}
if err := json.Unmarshal(configBytes, &effectiveConfig); err != nil {
t.Fatalf("failed to parse effective config metadata: %v", err)
@@ -3022,16 +3224,20 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if effectiveConfig.ValidationLLMConcurrency == nil || *effectiveConfig.ValidationLLMConcurrency != 1 {
t.Fatalf("expected validation_llm_concurrency=1 in effective config, got %#v", effectiveConfig.ValidationLLMConcurrency)
}
if effectiveConfig.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("expected transcript description in effective config, got %q", effectiveConfig.TranscriptDescription)
}
var invocation struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
TranscriptDescription string `json:"transcript_description"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
}
if err := json.Unmarshal(invocationBytes, &invocation); err != nil {
t.Fatalf("failed to parse invocation metadata: %v", err)
@@ -3051,6 +3257,9 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if invocation.ReportJSONPath != reportPath {
t.Fatalf("unexpected report_json_path: %q", invocation.ReportJSONPath)
}
if invocation.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("unexpected transcript_description: %q", invocation.TranscriptDescription)
}
if len(invocation.Modules) == 0 {
t.Fatalf("expected non-empty modules list in invocation metadata")
}

34
internal/core/config/config.go
View File

@@ -14,22 +14,23 @@ const (
)
const (
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
DefaultModulesCSV = "glossary,homophones,glossary,spoken_word,grammar"
DefaultPrimaryModel = "openrouter/google/gemma-4-31b-it"
DefaultPrimaryBaseURL = "https://openrouter.ai/api/v1"
DefaultPrimaryLLMTimeoutSeconds = 600
DefaultMaxRetries = 3
DefaultLLMConcurrency = 1
DefaultValidationMaxPromptTokens = 2048
DefaultMaxSectionTokens = 8192
DefaultMinSectionTokens = 2048
DefaultConfidenceThreshold = 0.8
DefaultNormalizeMaxSegmentGap = 4.0
DefaultNormalizeEllipsisGap = 3.5
DefaultNormalizeMaxSegmentDuration = 60.0
DefaultNormalizeMaxSegmentTokens = 2048
DefaultTranscriptDescriptionMaxChars = 500
DefaultWorkDir = "/tmp/audita"
DefaultWorkDirRetention WorkDirRetention = WorkDirRetentionAuto
)
type Config struct {
@@ -45,6 +46,7 @@ type Config struct {
TargetSections *int
Thresholds ConfidenceThresholds
Normalization NormalizationConfig
TranscriptDescription string
WorkDir string
WorkDirRetention WorkDirRetention
}

26
internal/core/config/config_test.go
View File

@@ -51,6 +51,9 @@ func TestDefaultConfigValues(t *testing.T) {
if cfg.WorkDir != DefaultWorkDir {
t.Fatalf("unexpected default work dir: %q", cfg.WorkDir)
}
if cfg.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", cfg.TranscriptDescription)
}
if cfg.WorkDirRetention != DefaultWorkDirRetention {
t.Fatalf("unexpected default work dir retention: %q", cfg.WorkDirRetention)
}
@@ -214,6 +217,29 @@ func TestApplyCLIOverridesPrecedence(t *testing.T) {
}
}
func TestApplyCLIOverridesTrimsTranscriptDescription(t *testing.T) {
cfg := Default()
description := " background context about speakers "
if err := cfg.ApplyCLIOverrides(CLIOverrides{TranscriptDescription: &description}); err != nil {
t.Fatalf("ApplyCLIOverrides failed: %v", err)
}
if cfg.TranscriptDescription != "background context about speakers" {
t.Fatalf("unexpected transcript description trim result: %q", cfg.TranscriptDescription)
}
}
func TestValidationRejectsOverlyLongTranscriptDescription(t *testing.T) {
cfg := Default()
cfg.TranscriptDescription = strings.Repeat("a", DefaultTranscriptDescriptionMaxChars+1)
err := cfg.Validate()
if err == nil {
t.Fatalf("expected transcript description length validation error")
}
if !strings.Contains(err.Error(), "transcript description must be 500 characters or fewer") {
t.Fatalf("unexpected validation error: %v", err)
}
}
func TestApplyCLIOverridesLegacyLLMConcurrencyAlias(t *testing.T) {
cfg := Default()
aliasConcurrency := 6

9
internal/core/config/flags.go
View File

@@ -1,6 +1,9 @@
package config
import "fmt"
import (
"fmt"
"strings"
)
type CLIOverrides struct {
ModulesCSV *string
@@ -30,6 +33,7 @@ type CLIOverrides struct {
NormalizeEllipsisGap *float64
NormalizeMaxSegmentDuration *float64
NormalizeMaxSegmentTokens *int
TranscriptDescription *string
WorkDir *string
WorkDirRetention *string
}
@@ -135,6 +139,9 @@ func (c *Config) ApplyCLIOverrides(overrides CLIOverrides) error {
if overrides.NormalizeMaxSegmentTokens != nil {
c.Normalization.MaxSegmentTokens = *overrides.NormalizeMaxSegmentTokens
}
if overrides.TranscriptDescription != nil {
c.TranscriptDescription = strings.TrimSpace(*overrides.TranscriptDescription)
}
if overrides.WorkDir != nil {
c.WorkDir = *overrides.WorkDir
}

3
internal/core/config/validation.go
View File

@@ -88,6 +88,9 @@ func (c Config) Validate() error {
if c.Normalization.MaxSegmentTokens <= 0 {
issues = append(issues, "normalize max segment tokens must be greater than zero")
}
if len(strings.TrimSpace(c.TranscriptDescription)) > DefaultTranscriptDescriptionMaxChars {
issues = append(issues, fmt.Sprintf("transcript description must be %d characters or fewer", DefaultTranscriptDescriptionMaxChars))
}
if strings.TrimSpace(c.WorkDir) == "" {
issues = append(issues, "work dir must not be empty")

17
internal/core/diagnostics/run_dir.go
View File

@@ -48,14 +48,15 @@ func ShouldRetainRunDirectory(input RetentionDecisionInput) bool {
// InvocationMetadata captures non-secret invocation details for diagnostics.
type InvocationMetadata struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path,omitempty"`
ReportJSONPath string `json:"report_json_path,omitempty"`
TranscriptDescription string `json:"transcript_description,omitempty"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt time.Time `json:"started_at"`
}
// NewRunDirectory creates a new run directory under the configured work dir

21
internal/framework/promptcontext/transcript_description.go Normal file
View File

@@ -0,0 +1,21 @@
package promptcontext
import (
"fmt"
"strings"
)
// TranscriptDescriptionBlock returns standardized background-only context
// guidance for prompt builders when a user-supplied description is present.
func TranscriptDescriptionBlock(transcriptDescription string) string {
description := strings.TrimSpace(transcriptDescription)
if description == "" {
return ""
}
return "Transcript description (background context only):\n" +
fmt.Sprintf("%s\n\n", description) +
"Use this description only as optional background to interpret ambiguous terms. " +
"It must not override the transcript content. " +
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.\n\n"
}

29
internal/framework/promptcontext/transcript_description_test.go Normal file
View File

@@ -0,0 +1,29 @@
package promptcontext
import (
"strings"
"testing"
)
func TestTranscriptDescriptionBlockEmpty(t *testing.T) {
if got := TranscriptDescriptionBlock(""); got != "" {
t.Fatalf("expected empty block for empty description, got %q", got)
}
if got := TranscriptDescriptionBlock(" "); got != "" {
t.Fatalf("expected empty block for whitespace description, got %q", got)
}
}
func TestTranscriptDescriptionBlockIncludesGuardrails(t *testing.T) {
got := TranscriptDescriptionBlock("Council hearing with multiple speakers.")
for _, want := range []string{
"Transcript description (background context only):",
"Council hearing with multiple speakers.",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(got, want) {
t.Fatalf("expected block to contain %q", want)
}
}
}

19
internal/framework/validators/llm_prompt_builders.go
View File

@@ -3,9 +3,11 @@ package validators
import (
"encoding/json"
"fmt"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -22,11 +24,12 @@ func BuildSpokenFormPlausibilityMessages(validationPayload []LLMValidationItem)
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildMeaningReversalMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -43,11 +46,12 @@ func BuildMeaningReversalMessages(validationPayload []LLMValidationItem) ([]LLMM
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
func BuildEditorialMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
payloadJSON, err := marshalPromptPayload(validationPayload)
if err != nil {
return nil, err
@@ -69,16 +73,17 @@ func BuildEditorialMessages(validationPayload []LLMValidationItem) ([]LLMMessage
"- If a correction includes categories, treat them as additional segment context.\n" +
"- Each returned validation must contain only correction_index, approved, confidence, and reason.\n" +
"- confidence must be between 0.0 and 1.0.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Corrections to validate:\n%s", payloadJSON)
return []LLMMessage{{Role: "system", Content: system}, {Role: "user", Content: user}}, nil
}
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildGrammarReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload)
func BuildSpokenWordReviewMessages(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error) {
return BuildEditorialMessages(validationPayload, transcriptDescription)
}
func marshalPromptPayload(validationPayload []LLMValidationItem) (string, error) {

8
internal/framework/validators/llm_validators.go
View File

@@ -13,7 +13,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
)
type LLMPromptBuilder func(validationPayload []LLMValidationItem) ([]LLMMessage, error)
type LLMPromptBuilder func(validationPayload []LLMValidationItem, transcriptDescription string) ([]LLMMessage, error)
type LLMBackedValidator struct {
name string
@@ -84,7 +84,11 @@ func (v *LLMBackedValidator) Validate(ctx context.Context, req Request) (Result,
llmDecisions := make([]Decision, 0)
for _, batch := range batches {
messages, err := v.promptBuilder(batch.Items)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := v.promptBuilder(batch.Items, transcriptDescription)
if err != nil {
return Result{}, err
}

25
internal/framework/validators/llm_validators_test.go
View File

@@ -167,7 +167,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters", Categories: []string{"narration"}}}
tests := []struct {
name string
build func([]LLMValidationItem) ([]LLMMessage, error)
build func([]LLMValidationItem, string) ([]LLMMessage, error)
mustHas []string
}{
{"spoken_form", BuildSpokenFormPlausibilityMessages, []string{"plausible spoken-form", "correction_index", "original_segment_text", "corrected_segment_text"}},
@@ -179,7 +179,7 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
msgs, err := tt.build(payload)
msgs, err := tt.build(payload, "Discussion among party members in a dungeon.")
if err != nil {
t.Fatalf("build err: %v", err)
}
@@ -192,10 +192,31 @@ func TestPromptBuildersContainRequiredContextAndInstructions(t *testing.T) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
for _, needle := range []string{
"Transcript description (background context only):",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, needle) {
t.Fatalf("expected prompt to contain %q", needle)
}
}
})
}
}
func TestPromptBuildersOmitTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
payload := []LLMValidationItem{{CorrectionIndex: 0, SegmentID: 1, OriginalText: "gestures", CorrectedText: "Jesters", OriginalSegmentText: "There were gestures", CorrectedSegmentText: "There were Jesters"}}
msgs, err := BuildSpokenFormPlausibilityMessages(payload, " ")
if err != nil {
t.Fatalf("build err: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestLLMBackedValidatorApprovalAndRejection(t *testing.T) {
client := &fakeStructuredLLMClient{responses: []LLMValidationResponse{{Validations: []LLMValidationDecision{
{CorrectionIndex: 0, Approved: true, Confidence: 0.9, Reason: "ok"},

6
internal/modules/glossary/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

20
internal/modules/glossary/module_test.go
View File

@@ -61,7 +61,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -91,6 +91,24 @@ func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Campaign scene in a crowded harbor.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
for _, want := range []string{
"Transcript description (background context only):",
"Campaign scene in a crowded harbor.",
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(combined, want) {
t.Fatalf("expected prompt to contain %q", want)
}
}
}
func TestGlossaryModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/glossary/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -22,7 +23,7 @@ type promptTranscriptSection struct {
Segments []promptSegment `json:"segments"`
}
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -71,6 +72,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Glossary:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/grammar/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

16
internal/modules/grammar/module_test.go
View File

@@ -71,7 +71,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -97,6 +97,20 @@ func TestBuildProposalMessagesConstraints(t *testing.T) {
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Courtroom exchange with formal titles.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "must not override the transcript content") {
t.Fatalf("expected no-override guidance in prompt")
}
}
func TestGrammarModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/grammar/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to punctuation/capitalization/
// spacing cleanup with strict meaning guards.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -76,6 +77,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/homophones/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

27
internal/modules/homophones/module_test.go
View File

@@ -62,7 +62,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -100,6 +100,31 @@ func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testi
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "must not override the transcript content") {
t.Fatalf("expected no-override guidance in prompt")
}
}
func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("did not expect empty transcript description section in prompt")
}
}
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/homophones/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to conservative homophone and
// mistranscription updates.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -74,6 +75,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{

6
internal/modules/spoken_word/module.go
View File

@@ -55,7 +55,11 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) ([]
if req.Section != nil {
sectionIndex = req.Section.Index
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex)
transcriptDescription := ""
if req.Config != nil {
transcriptDescription = req.Config.TranscriptDescription
}
messages, err := BuildProposalMessages(sectionTranscript, req.Glossary, sectionIndex, transcriptDescription)
if err != nil {
return nil, err
}

16
internal/modules/spoken_word/module_test.go
View File

@@ -62,7 +62,7 @@ func tinyGlossary() *schema.Glossary {
}
func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0)
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
@@ -99,6 +99,20 @@ func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T)
}
}
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Participants are discussing raid logistics.")
if err != nil {
t.Fatalf("BuildProposalMessages error: %v", err)
}
combined := msgs[0].Content + "\n" + msgs[1].Content
if !strings.Contains(combined, "Transcript description (background context only):") {
t.Fatalf("expected transcript description section in prompt")
}
if !strings.Contains(combined, "Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.") {
t.Fatalf("expected anti-invention guidance in prompt")
}
}
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
m, err := New()
if err != nil {

4
internal/modules/spoken_word/prompt.go
View File

@@ -6,6 +6,7 @@ import (
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
)
type promptSegment struct {
@@ -24,7 +25,7 @@ type promptTranscriptSection struct {
// BuildProposalMessages constrains corrections to conservative dysfluency
// cleanup with strict semantic preservation.
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int) ([]contracts.LLMMessage, error) {
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
if err != nil {
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
@@ -77,6 +78,7 @@ func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Gloss
"- Return only changed segments; do not return entries for unchanged segments.\n" +
"- confidence must be between 0.0 and 1.0.\n" +
"- If no corrections are needed, return an empty corrections list.\n\n" +
promptcontext.TranscriptDescriptionBlock(transcriptDescription) +
fmt.Sprintf("Protected glossary/context:\n%s\n\nTranscript section:\n%s", string(glossaryJSON), string(sectionJSON))
return []contracts.LLMMessage{