eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add transcript description prompt context

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-13 12:13:32 +00:00
parent de99467ede
commit ebbd2c8a63
27 changed files with 542 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
internal/cli/run.go
View File

@@ -74,12 +74,13 @@ var processRunner = func(inv processInvocation, stdout io.Writer) (*normalizatio
}
if err := runDir.WriteInvocationMetadata(diagnostics.InvocationMetadata{
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
Modules: append([]string(nil), inv.Config.Modules...),
Operation: "process",
TranscriptPath: inv.TranscriptPath,
GlossaryPath: inv.GlossaryPath,
OutputPath: inv.OutputPath,
ReportJSONPath: inv.ReportJSONPath,
TranscriptDescription: inv.Config.TranscriptDescription,
Modules: append([]string(nil), inv.Config.Modules...),
}); err != nil {
_ = runDir.WriteErrorLog(fmt.Sprintf("invocation_metadata: %v", err))
}
@@ -435,6 +436,8 @@ func runProcess(args []string, stdout, stderr io.Writer) int {
overrides.NormalizeMaxSegmentDuration = pFlags.normalizeMaxSegmentDuration
case "normalize-max-segment-tokens":
overrides.NormalizeMaxSegmentTokens = pFlags.normalizeMaxSegmentTokens
case "transcript-description":
overrides.TranscriptDescription = pFlags.transcriptDescription
case "work-dir":
overrides.WorkDir = pFlags.workDir
case "work-dir-retention":
@@ -710,6 +713,7 @@ type processFlags struct {
normalizeEllipsisGap *float64
normalizeMaxSegmentDuration *float64
normalizeMaxSegmentTokens *int
transcriptDescription *string
workDir *string
workDirRetention *string
}
@@ -769,6 +773,7 @@ func newProcessFlagSet(cfg config.Config, stderr io.Writer) (*flag.FlagSet, proc
normalizeEllipsisGap: fs.Float64("normalize-ellipsis-gap", cfg.Normalization.EllipsisGap, "Gap threshold for ellipsis insertion"),
normalizeMaxSegmentDuration: fs.Float64("normalize-max-segment-duration", cfg.Normalization.MaxSegmentDuration, "Maximum merged segment duration"),
normalizeMaxSegmentTokens: fs.Int("normalize-max-segment-tokens", cfg.Normalization.MaxSegmentTokens, "Maximum merged segment token estimate"),
transcriptDescription: fs.String("transcript-description", cfg.TranscriptDescription, "Brief background context for LLM prompts; does not override transcript content"),
workDir: fs.String("work-dir", cfg.WorkDir, "Per-run work directory"),
workDirRetention: fs.String("work-dir-retention", string(cfg.WorkDirRetention), "Work-dir retention policy: auto|always|never"),
}

231
internal/cli/run_test.go
View File

@@ -15,6 +15,7 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/normalization"
"gitea.maximumdirect.net/eric/audita/internal/core/reporting"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -83,6 +84,7 @@ func TestRunProcessHelpListsExpectedFlags(t *testing.T) {
"--normalize-ellipsis-gap",
"--normalize-max-segment-duration",
"--normalize-max-segment-tokens",
"--transcript-description",
"--work-dir",
"--work-dir-retention",
} {
@@ -249,6 +251,105 @@ func TestRunProcessCLIOverridesEnvironment(t *testing.T) {
}
}
func TestRunProcessTranscriptDescriptionDefaultEmpty(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "" {
t.Fatalf("expected default transcript description to be empty, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessTranscriptDescriptionCLIOverrideAndTrim(t *testing.T) {
processModuleFactory = fakeModuleFactory{modules: map[string]contracts.TranscriptModule{
"m": fakeModule{
key: "m",
policy: proposals.ReplacementPolicyRequireUnique,
validators: []contracts.Validator{
fakeValidator{name: "capture-config", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
if req.Config == nil {
t.Fatal("expected config in validation request")
}
if req.Config.TranscriptDescription != "speaker background context" {
t.Fatalf("expected trimmed transcript description, got %q", req.Config.TranscriptDescription)
}
return validators.Result{ValidatorName: "capture-config", Decisions: nil}, nil
}},
},
proposeF: func(req contracts.ProposalRequest) ([]proposals.CorrectionProposal, error) { return nil, nil },
},
}}
t.Cleanup(func() { processModuleFactory = nil })
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "m",
"--transcript-description", " speaker background context ",
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
}
func TestRunProcessRejectsOverlyLongTranscriptDescription(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"Hello"}
]`)
tooLong := strings.Repeat("a", config.DefaultTranscriptDescriptionMaxChars+1)
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--transcript-description", tooLong,
}, &stdout, &stderr)
if exitCode == 0 {
t.Fatalf("expected nonzero exit code for overly long transcript description")
}
if stdout.Len() != 0 {
t.Fatalf("expected empty stdout, got %q", stdout.String())
}
if !strings.Contains(stderr.String(), "transcript description must be 500 characters or fewer") {
t.Fatalf("expected transcript description length validation error, got %q", stderr.String())
}
}
func TestRunProcessRejectsValidationConcurrencyAboveTotalConcurrency(t *testing.T) {
var stdout bytes.Buffer
var stderr bytes.Buffer
@@ -1091,6 +1192,35 @@ func (f *fakeStructuredLLMClient) CompleteStructured(ctx context.Context, req co
}
}
type capturePromptStructuredLLMClient struct {
proposalResponses []proposal_generation.StructuredCorrectionSet
validationResponses []validators.LLMValidationResponse
requests []contracts.StructuredCompletionRequest
}
func (c *capturePromptStructuredLLMClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
_ = ctx
c.requests = append(c.requests, req)
switch target := out.(type) {
case *proposal_generation.StructuredCorrectionSet:
if len(c.proposalResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected proposal llm call")
}
*target = c.proposalResponses[0]
c.proposalResponses = c.proposalResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
case *validators.LLMValidationResponse:
if len(c.validationResponses) == 0 {
return contracts.StructuredCompletionResponse{}, errors.New("unexpected validation llm call")
}
*target = c.validationResponses[0]
c.validationResponses = c.validationResponses[1:]
return contracts.StructuredCompletionResponse{}, nil
default:
return contracts.StructuredCompletionResponse{}, errors.New("unexpected llm output type")
}
}
func TestRunProcessInjectedFactoryExecutesRunnerAndReportsModules(t *testing.T) {
allow := fakeValidator{name: "allow", validateF: func(req contracts.ValidationRequest) (validators.Result, error) {
decisions := make([]validators.Decision, len(req.CandidateProposal))
@@ -1466,6 +1596,75 @@ func TestRunProcessExplicitGrammarAppliesCorrectionAndReportsDiagnostics(t *test
}
}
func TestRunProcessTranscriptDescriptionReachesProposalAndValidatorPrompts(t *testing.T) {
proposalClient := &capturePromptStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
{Corrections: []proposal_generation.StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "hello ,world", CorrectedText: "Hello, world", Confidence: 0.95},
}},
},
}
validationClient := &capturePromptStructuredLLMClient{
validationResponses: []validators.LLMValidationResponse{
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
{Validations: []validators.LLMValidationDecision{{CorrectionIndex: 0, Approved: true, Confidence: 0.99, Reason: "ok"}}},
},
}
processProposalLLMClient = proposalClient
processValidationLLMClient = validationClient
t.Cleanup(func() {
processProposalLLMClient = nil
processValidationLLMClient = nil
processProposalLLMScheduler = nil
processValidationLLMScheduler = nil
})
var stdout, stderr bytes.Buffer
transcriptPath := writeFile(t, "transcript.json", `[
{"id":1,"speaker":"Alice","start":0.0,"end":1.0,"text":"hello ,world"}
]`)
description := "Hearing transcript where speakers reference proper nouns."
exitCode := Run([]string{
"process", transcriptPath,
"--glossary", fixturePath("tiny_glossary.yaml"),
"--modules", "grammar",
"--transcript-description", description,
}, &stdout, &stderr)
if exitCode != 0 {
t.Fatalf("expected success, got %d stderr=%q", exitCode, stderr.String())
}
if len(proposalClient.requests) == 0 {
t.Fatalf("expected proposal LLM requests")
}
if len(validationClient.requests) == 0 {
t.Fatalf("expected validation LLM requests")
}
proposalPrompt := combinedPrompt(proposalClient.requests[0].Messages)
validatorPrompt := combinedPrompt(validationClient.requests[0].Messages)
for _, prompt := range []string{proposalPrompt, validatorPrompt} {
for _, want := range []string{
"Transcript description (background context only):",
description,
"must not override the transcript content",
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
} {
if !strings.Contains(prompt, want) {
t.Fatalf("expected prompt to contain %q, got: %q", want, prompt)
}
}
}
}
func combinedPrompt(messages []contracts.LLMMessage) string {
parts := make([]string, 0, len(messages))
for _, m := range messages {
parts = append(parts, m.Content)
}
return strings.Join(parts, "\n")
}
func TestRunProcessExplicitGrammarRejectedAndApplicationSkipAreDistinct(t *testing.T) {
processProposalLLMClient = &fakeStructuredLLMClient{
proposalResponses: []proposal_generation.StructuredCorrectionSet{
@@ -2957,6 +3156,8 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
fixturePath("tiny_transcript.json"),
"--glossary",
fixturePath("tiny_glossary.yaml"),
"--transcript-description",
" scene takes place during a council hearing ",
"--total-llm-concurrency",
"3",
"--proposal-llm-concurrency",
@@ -3006,9 +3207,10 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
}
var effectiveConfig struct {
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TotalLLMConcurrency int `json:"TotalLLMConcurrency"`
ProposalLLMConcurrency int `json:"ProposalLLMConcurrency"`
ValidationLLMConcurrency *int `json:"ValidationLLMConcurrency"`
TranscriptDescription string `json:"TranscriptDescription"`
}
if err := json.Unmarshal(configBytes, &effectiveConfig); err != nil {
t.Fatalf("failed to parse effective config metadata: %v", err)
@@ -3022,16 +3224,20 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if effectiveConfig.ValidationLLMConcurrency == nil || *effectiveConfig.ValidationLLMConcurrency != 1 {
t.Fatalf("expected validation_llm_concurrency=1 in effective config, got %#v", effectiveConfig.ValidationLLMConcurrency)
}
if effectiveConfig.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("expected transcript description in effective config, got %q", effectiveConfig.TranscriptDescription)
}
var invocation struct {
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
Operation string `json:"operation"`
TranscriptPath string `json:"transcript_path"`
GlossaryPath string `json:"glossary_path"`
OutputPath string `json:"output_path"`
ReportJSONPath string `json:"report_json_path"`
TranscriptDescription string `json:"transcript_description"`
Modules []string `json:"modules"`
RunID string `json:"run_id"`
StartedAt string `json:"started_at"`
}
if err := json.Unmarshal(invocationBytes, &invocation); err != nil {
t.Fatalf("failed to parse invocation metadata: %v", err)
@@ -3051,6 +3257,9 @@ func TestRunProcessWritesRedactedRunMetadataArtifacts(t *testing.T) {
if invocation.ReportJSONPath != reportPath {
t.Fatalf("unexpected report_json_path: %q", invocation.ReportJSONPath)
}
if invocation.TranscriptDescription != "scene takes place during a council hearing" {
t.Fatalf("unexpected transcript_description: %q", invocation.TranscriptDescription)
}
if len(invocation.Modules) == 0 {
t.Fatalf("expected non-empty modules list in invocation metadata")
}