Route proposal modules through embedded PromptKit definitions
This commit is contained in:
@@ -35,10 +35,14 @@ Shared proposal-generation plumbing is centralized in:
|
||||
- `internal/framework/proposal_generation`
|
||||
|
||||
Module packages own:
|
||||
- prompt selection (`internal/prompts` prompt IDs);
|
||||
- module-specific prompt payload construction.
|
||||
- prompt selection (`internal/prompts` prompt IDs).
|
||||
|
||||
Shared prompt helpers live in `internal/framework/promptcontext`.
|
||||
`proposal_generation.BuildProposalVariables` encodes the glossary, selected
|
||||
transcript section, and trimmed description once for all four modules. PromptKit
|
||||
renders their embedded definitions from those variables and validates the
|
||||
correction-set schema before Audita maps proposals. Domain validators still
|
||||
check confidence, applicability, protected glossary terms, and replacement
|
||||
policy. Shared context encoding lives in `internal/framework/promptcontext`.
|
||||
|
||||
## Validator Chain Ownership
|
||||
|
||||
@@ -48,21 +52,21 @@ Module packages call the built-in chain resolver at construction.
|
||||
## Failure and Warning Behavior
|
||||
|
||||
Module construction failures propagate to orchestration; execution failures can
|
||||
preserve partial module results. Malformed structured proposal payloads become
|
||||
warnings and section-level rejection. Consumer-visible outcomes are defined by
|
||||
preserve partial module results. Exhausted structural repair becomes a warning
|
||||
and section-level rejection; provider and protocol failures propagate. Consumer-visible outcomes are defined by
|
||||
the [report contract](../integrations/output-files.md#module-outcomes).
|
||||
|
||||
## Changing Modules
|
||||
|
||||
1. Add or update the narrowly scoped package under `internal/modules/<module_key>`.
|
||||
2. Keep module-specific prompt ownership in the module and `internal/prompts`;
|
||||
see [prompt rendering and metadata](prompts.md).
|
||||
2. Keep module prompt selection in the module and its definition in
|
||||
`internal/prompts`; see [prompt assets](prompts.md).
|
||||
3. Wire module registration and catalog resolution through framework module
|
||||
registration and the core module catalog. Update selectable keys in the
|
||||
[configuration contract](../config.md) when they change.
|
||||
4. Verify replacement policy and validator-chain selection against the
|
||||
[architecture requirements](../policy/architecture.md#correction-and-concurrency-invariants).
|
||||
5. Review module, prompt-rendering, and proposal-generation tests. Inspect parity
|
||||
5. Review module, prompt-variable, and proposal-generation tests. Inspect parity
|
||||
and release fixtures when public output changes, choosing sufficient coverage
|
||||
under the [testing policy](../policy/testing.md).
|
||||
6. Update this document for implementation changes and the appropriate public
|
||||
|
||||
@@ -312,7 +312,16 @@ PromptKit's entire status-code matrix.
|
||||
|
||||
## Stage 6: Migrate All Proposal Callers
|
||||
|
||||
**Status:** Pending. **Depends on:** Stage 5.
|
||||
**Status:** Complete (2026-09-24). All four modules select embedded prompt IDs
|
||||
and share domain variable encoding; proposal generation passes the correction-set
|
||||
key and diagnostic context through the prepared client, retaining ordering,
|
||||
warnings, and later domain validation. Updated proposal and module tests and
|
||||
module internals documentation. The isolated variable-encoding test passed with
|
||||
`go test internal/framework/proposal_generation/variables.go internal/framework/proposal_generation/variables_test.go`;
|
||||
`git diff --check` passed. Assembled proposal/module tests are deferred because
|
||||
Stage 7's validator imports still reference removed message-rendering types.
|
||||
The repaired-proposal/domain integration check remains deferred with those imports.
|
||||
**Depends on:** Stage 5.
|
||||
**Owns:** `internal/modules`, `internal/framework/proposal_generation`, and their tests.
|
||||
|
||||
1. Switch glossary, homophones, spoken_word, and grammar proposal construction to
|
||||
|
||||
@@ -56,9 +56,9 @@ type Request struct {
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
Glossary *schema.Glossary `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
Messages []contracts.LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PromptMetadata map[string]any `json:"prompt_metadata,omitempty"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version"`
|
||||
Variables map[string]string `json:"variables"`
|
||||
StageName string `json:"stage_name,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
@@ -90,16 +90,14 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("structured LLM client is required")
|
||||
}
|
||||
if len(req.Messages) == 0 {
|
||||
return Result{}, fmt.Errorf("messages must not be empty")
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return Result{}, fmt.Errorf("prompt ID must not be empty")
|
||||
}
|
||||
|
||||
stage := strings.TrimSpace(req.StageName)
|
||||
if stage == "" {
|
||||
stage = stagename.ProposalGeneration(req.ModuleInstance, sectionIndexPtr(req.Section))
|
||||
}
|
||||
model := resolveModel(req.Config, req.Model)
|
||||
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
||||
|
||||
var writer InteractionDiagnosticsWriter
|
||||
if req.DiagnosticsWriter != nil {
|
||||
@@ -114,17 +112,16 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}
|
||||
|
||||
var (
|
||||
response StructuredCorrectionSet
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
response StructuredCorrectionSet
|
||||
completion contracts.StructuredCompletionResponse
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
)
|
||||
responseSchema := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: model,
|
||||
ResponseSchema: &responseSchema,
|
||||
completion, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
|
||||
PromptID: req.PromptID, PromptVersion: req.PromptVersion,
|
||||
Variables: req.Variables, Schema: responseschema.CorrectionSetKey,
|
||||
Context: structuredoutput.DiagnosticContext{Role: "proposal", ModuleKey: req.ModuleKey, ModuleInstance: req.ModuleInstance, SectionIndex: sectionIndexPtr(req.Section)},
|
||||
}, &response)
|
||||
return callErr
|
||||
}
|
||||
@@ -140,21 +137,16 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"model": model,
|
||||
"prompt": completion.Prompt,
|
||||
}
|
||||
if len(req.PromptMetadata) > 0 {
|
||||
requestMetadata["prompt_metadata"] = req.PromptMetadata
|
||||
}
|
||||
requestMetadata["response_schema"] = responseSchema.DiagnosticsMap()
|
||||
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
|
||||
|
||||
if writer != nil {
|
||||
artifacts, _ = writer.WriteInteraction(
|
||||
stage,
|
||||
requestMetadata,
|
||||
map[string]any{
|
||||
"messages": messages,
|
||||
},
|
||||
response,
|
||||
completion.Prepared,
|
||||
map[string]any{"decoded": response, "final_content": completion.FinalContent, "model": completion.Model},
|
||||
errPayload(callErr),
|
||||
)
|
||||
}
|
||||
@@ -203,16 +195,6 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func resolveModel(cfg *config.Config, override string) string {
|
||||
if strings.TrimSpace(override) != "" {
|
||||
return strings.TrimSpace(override)
|
||||
}
|
||||
if cfg == nil {
|
||||
return ""
|
||||
}
|
||||
return llm.ResolvePrimaryConfig(*cfg).Model
|
||||
}
|
||||
|
||||
func errPayload(err error) any {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
@@ -14,11 +14,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/responseschema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/structuredoutput"
|
||||
)
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
@@ -42,7 +44,11 @@ func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contr
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
var content strings.Builder
|
||||
for _, value := range req.Variables {
|
||||
content.WriteString(value)
|
||||
}
|
||||
return contracts.StructuredCompletionResponse{Prepared: &structuredoutput.PreparedSnapshot{Messages: []structuredoutput.Message{{Role: "user", Content: content.String()}}}}, nil
|
||||
}
|
||||
|
||||
type countingScheduler struct {
|
||||
@@ -116,11 +122,10 @@ func defaultRequest(t *testing.T) Request {
|
||||
ModuleInstance: "test_module",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Config: &cfg,
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
{Role: "user", Content: "user prompt"},
|
||||
},
|
||||
StartIndex: 0,
|
||||
PromptID: "modules.grammar.proposal",
|
||||
PromptVersion: "v1",
|
||||
Variables: map[string]string{"SectionJSON": "[]"},
|
||||
StartIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,12 +177,8 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
|
||||
t.Fatalf("expected 1 LLM call, got %d", len(client.calls))
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.ResponseSchema == nil {
|
||||
t.Fatalf("expected response schema on structured request")
|
||||
}
|
||||
want := responseschema.MustLookup(responseschema.CorrectionSetKey)
|
||||
if call.ResponseSchema.ID != want.ID || call.ResponseSchema.Version != want.Version || call.ResponseSchema.Name != want.Name || call.ResponseSchema.SHA256 != want.SHA256 {
|
||||
t.Fatalf("unexpected response schema metadata: got=%+v want=%+v", *call.ResponseSchema, want)
|
||||
if call.Schema != responseschema.CorrectionSetKey || call.PromptID != req.PromptID || call.Context.Role != "proposal" {
|
||||
t.Fatalf("unexpected call contract: %+v", call)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,13 +192,6 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsWriter = diag
|
||||
req.PromptMetadata = map[string]any{
|
||||
"prompt_id": "modules.grammar.proposal",
|
||||
"prompt_version": "v1",
|
||||
"prompt_source": "builtin",
|
||||
"embedded_path": "assets/modules/grammar",
|
||||
"sha256": "abc",
|
||||
}
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
@@ -215,13 +209,7 @@ func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
|
||||
if schemaMap["id"] != want.ID || schemaMap["version"] != want.Version || schemaMap["name"] != want.Name || schemaMap["sha256"] != want.SHA256 {
|
||||
t.Fatalf("unexpected diagnostics schema metadata: got=%v want=%+v", schemaMap, want)
|
||||
}
|
||||
promptMap, ok := metadata["prompt_metadata"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected prompt_metadata map, got %T", metadata["prompt_metadata"])
|
||||
}
|
||||
if promptMap["prompt_id"] != "modules.grammar.proposal" || promptMap["prompt_version"] != "v1" || promptMap["prompt_source"] != "builtin" || promptMap["embedded_path"] != "assets/modules/grammar" || promptMap["sha256"] != "abc" {
|
||||
t.Fatalf("unexpected prompt metadata map: %v", promptMap)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *testing.T) {
|
||||
@@ -253,7 +241,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("malformed structured output")}
|
||||
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureMalformedOutput, Message: "malformed structured output"}}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
@@ -272,20 +260,13 @@ func TestGenerateCandidatesMalformedStructuredOutputReturnsWarning(t *testing.T)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesProviderMalformedEnvelopeReturnsWarning(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("provider response missing choices")}
|
||||
func TestGenerateCandidatesProviderMalformedEnvelopePropagates(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: &structuredoutput.Failure{Kind: structuredoutput.FailureRuntime, Message: "structured model call failed", Cause: errors.New("malformed provider envelope")}}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
result, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("expected malformed provider envelope to downgrade to warning, got %v", err)
|
||||
}
|
||||
if len(result.Corrections) != 0 || len(result.Enriched) != 0 {
|
||||
t.Fatalf("expected no proposals on malformed response, got %+v", result)
|
||||
}
|
||||
if len(result.Warnings) != 1 || result.Warnings[0].ReasonCode != "proposal_response_malformed" {
|
||||
t.Fatalf("unexpected warnings: %+v", result.Warnings)
|
||||
if err == nil || len(result.Warnings) != 0 {
|
||||
t.Fatalf("protocol failure was downgraded: %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,17 +349,18 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = primarySecret
|
||||
cfg.ValidationLLM.APIKey = validationSecret
|
||||
t.Setenv("PRIMARY_SECRET", primarySecret)
|
||||
t.Setenv("VALIDATION_SECRET", validationSecret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n primary:\n endpoint: https://primary.invalid/v1\n credential: PRIMARY_SECRET\n validation:\n endpoint: https://validation.invalid/v1\n credential: VALIDATION_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
req := defaultRequest(t)
|
||||
req.Config = &cfg
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
req.Messages = []contracts.LLMMessage{
|
||||
{Role: "system", Content: "include secret " + primarySecret},
|
||||
{Role: "user", Content: "fix it"},
|
||||
}
|
||||
req.Variables = map[string]string{"SectionJSON": "include secret " + primarySecret}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
|
||||
@@ -11,19 +11,19 @@ import (
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type ProposalMessageBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error)
|
||||
type ProposalVariableBuilder func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error)
|
||||
|
||||
type ModuleProposalRequest struct {
|
||||
ProposalRequest contracts.ProposalRequest
|
||||
PromptID string
|
||||
BuildMessages ProposalMessageBuilder
|
||||
BuildVariables ProposalVariableBuilder
|
||||
}
|
||||
|
||||
// ExecuteModuleProposal runs shared proposal generation plumbing for one
|
||||
// module, leaving only module-specific prompt message building at call sites.
|
||||
func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (contracts.ProposalResult, error) {
|
||||
if req.BuildMessages == nil {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("proposal message builder is required")
|
||||
if req.BuildVariables == nil {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("proposal variable builder is required")
|
||||
}
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return contracts.ProposalResult{}, fmt.Errorf("prompt ID must not be empty")
|
||||
@@ -36,10 +36,10 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
|
||||
transcriptDescription := ""
|
||||
if req.ProposalRequest.Config != nil {
|
||||
transcriptDescription = req.ProposalRequest.Config.TranscriptDescription
|
||||
transcriptDescription = req.ProposalRequest.Config.Context.Description
|
||||
}
|
||||
|
||||
messages, err := req.BuildMessages(
|
||||
variables, err := req.BuildVariables(
|
||||
req.ProposalRequest.WorkingTranscript,
|
||||
req.ProposalRequest.Glossary,
|
||||
sectionIndex,
|
||||
@@ -62,8 +62,9 @@ func ExecuteModuleProposal(ctx context.Context, req ModuleProposalRequest) (cont
|
||||
Section: req.ProposalRequest.Section,
|
||||
Glossary: req.ProposalRequest.Glossary,
|
||||
Config: req.ProposalRequest.Config,
|
||||
Messages: messages,
|
||||
PromptMetadata: promptMetadata.DiagnosticsMap(),
|
||||
PromptID: promptMetadata.PromptID,
|
||||
PromptVersion: promptMetadata.PromptVersion,
|
||||
Variables: variables,
|
||||
StageName: stagename.ModuleProposal(req.ProposalRequest.RunSpec.InstanceName, sectionIndexPtr(req.ProposalRequest.Section)),
|
||||
StartIndex: 0,
|
||||
LLMClient: req.ProposalRequest.LLMClient,
|
||||
|
||||
@@ -20,7 +20,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 7}
|
||||
cfg := config.Default()
|
||||
cfg.TranscriptDescription = "Hearing transcript with role titles."
|
||||
cfg.Context.Description = "Hearing transcript with role titles."
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "teh"}}}
|
||||
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "X"}}}
|
||||
|
||||
@@ -45,12 +45,12 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
LLMClient: client,
|
||||
},
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||
BuildMessages: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
BuildVariables: func(inTranscript *schema.Transcript, inGlossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||
gotSectionIndex = sectionIndex
|
||||
gotDescription = transcriptDescription
|
||||
gotTranscript = inTranscript
|
||||
gotGlossary = inGlossary
|
||||
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||
return map[string]string{"SectionJSON": "[]"}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -59,8 +59,8 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
if gotSectionIndex != 7 {
|
||||
t.Fatalf("section index: got=%d want=%d", gotSectionIndex, 7)
|
||||
}
|
||||
if gotDescription != cfg.TranscriptDescription {
|
||||
t.Fatalf("transcript description: got=%q want=%q", gotDescription, cfg.TranscriptDescription)
|
||||
if gotDescription != cfg.Context.Description {
|
||||
t.Fatalf("transcript description: got=%q want=%q", gotDescription, cfg.Context.Description)
|
||||
}
|
||||
if gotTranscript != transcript {
|
||||
t.Fatalf("expected shared transcript pointer")
|
||||
@@ -68,7 +68,7 @@ func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing
|
||||
if gotGlossary != glossary {
|
||||
t.Fatalf("expected shared glossary pointer")
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal:section-0007" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal || client.calls[0].Context.SectionIndex == nil || *client.calls[0].Context.SectionIndex != 7 {
|
||||
t.Fatalf("unexpected stage name calls: %+v", client.calls)
|
||||
}
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "the" {
|
||||
@@ -82,7 +82,7 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
@@ -92,8 +92,8 @@ func TestExecuteModuleProposalValidatesInputs(t *testing.T) {
|
||||
|
||||
_, err = ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
PromptID: "missing.prompt.id",
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
return []contracts.LLMMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}, nil
|
||||
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||
return map[string]string{"SectionJSON": "[]"}, nil
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
@@ -105,7 +105,7 @@ func TestExecuteModuleProposalPropagatesBuilderError(t *testing.T) {
|
||||
wantErr := errors.New("builder failed")
|
||||
_, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||
BuildMessages: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
BuildVariables: func(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||
return nil, wantErr
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,37 +1,27 @@
|
||||
package glossary
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
// BuildProposalVariables preserves the domain inputs while PromptKit renders
|
||||
// the selected embedded proposal definition.
|
||||
func BuildProposalVariables(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) (map[string]string, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleGlossaryProposal, map[string]string{
|
||||
return map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
27
internal/framework/proposal_generation/variables_test.go
Normal file
27
internal/framework/proposal_generation/variables_test.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestBuildProposalVariablesPreservesSectionGlossaryAndDescription(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 1, Speaker: "A", Start: 0, End: 1, Text: "gestures"}}}
|
||||
glossary := &schema.Glossary{Entries: []schema.GlossaryEntry{{Name: "Jesters"}}}
|
||||
variables, err := BuildProposalVariables(transcript, glossary, 0, " Fantasy names ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(variables["SectionJSON"], `"section_index": 0`) || !strings.Contains(variables["SectionJSON"], "gestures") || !strings.Contains(variables["GlossaryJSON"], "Jesters") || !strings.Contains(variables["TranscriptDescriptionBlock"], "Fantasy names") {
|
||||
t.Fatalf("encoded variables incomplete: %+v", variables)
|
||||
}
|
||||
empty, err := BuildProposalVariables(transcript, glossary, 0, " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if empty["TranscriptDescriptionBlock"] != "" {
|
||||
t.Fatal("empty description added guidance")
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -60,55 +62,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Glossary:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Jester"`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"glossary-supported corrections",
|
||||
"acoustically similar",
|
||||
"Do not make generic grammar, spelling, capitalization, style, or filler-word edits",
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(combined), "summarize the transcript") {
|
||||
t.Fatalf("prompt should not invite summarization")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Campaign scene in a crowded harbor.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Transcript description (background context only):",
|
||||
"Campaign scene in a crowded harbor.",
|
||||
"must not override the transcript content",
|
||||
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlossaryModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -155,8 +108,12 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -183,9 +140,14 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGlossaryProposal {
|
||||
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -70,47 +72,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Transcript section:",
|
||||
"Protected glossary/context:",
|
||||
`"section_index": 0`,
|
||||
"punctuation, capitalization, spacing, and article cleanup only",
|
||||
"Do not make word substitutions",
|
||||
"Do not return speaker, start, or end fields",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(combined), "summarize") {
|
||||
t.Fatalf("prompt should not invite summarization")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Courtroom exchange with formal titles.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "must not override the transcript content") {
|
||||
t.Fatalf("expected no-override guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrammarModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -157,8 +118,12 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
module, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -185,9 +150,14 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal {
|
||||
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package grammar
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to punctuation/capitalization/
|
||||
// spacing cleanup with strict meaning guards.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleGrammarProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleHomophonesProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -61,70 +63,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Protected glossary/context:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"homophone correction assistant",
|
||||
"mistranscriptions of spoken English",
|
||||
"Do not add or remove punctuation",
|
||||
"Do not return speaker, start, or end fields",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"summarize the transcript",
|
||||
"style rewrite",
|
||||
"grammar cleanup",
|
||||
"punctuation cleanup",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(combined), forbidden) {
|
||||
t.Fatalf("prompt should not invite %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "must not override the transcript content") {
|
||||
t.Fatalf("expected no-override guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("did not expect empty transcript description section in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -171,8 +109,12 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -199,9 +141,14 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleHomophonesProposal {
|
||||
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package homophones
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative homophone and
|
||||
// mistranscription updates.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleHomophonesProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleSpokenWordProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -61,57 +63,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Protected glossary/context:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"conservative spoken-word cleanup assistant",
|
||||
"Preserve substantive meaning",
|
||||
"Do not paraphrase, summarize, reorder ideas",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"style rewriting",
|
||||
"grammar-only cleanup",
|
||||
"punctuation-only cleanup",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(combined), forbidden) {
|
||||
t.Fatalf("prompt should not invite %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Participants are discussing raid logistics.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.") {
|
||||
t.Fatalf("expected anti-invention guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -158,8 +109,12 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -186,9 +141,14 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleSpokenWordProposal {
|
||||
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package spoken_word
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative dysfluency
|
||||
// cleanup with strict semantic preservation.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleSpokenWordProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user