Route proposal modules through embedded PromptKit definitions
This commit is contained in:
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGlossaryProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -60,55 +62,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsGlossaryContextAndConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Glossary:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Jester"`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"glossary-supported corrections",
|
||||
"acoustically similar",
|
||||
"Do not make generic grammar, spelling, capitalization, style, or filler-word edits",
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(combined), "summarize the transcript") {
|
||||
t.Fatalf("prompt should not invite summarization")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Campaign scene in a crowded harbor.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Transcript description (background context only):",
|
||||
"Campaign scene in a crowded harbor.",
|
||||
"must not override the transcript content",
|
||||
"Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.",
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGlossaryModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -155,8 +108,12 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -183,9 +140,14 @@ func TestGlossaryModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T)
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "glossary:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGlossaryProposal {
|
||||
t.Fatalf("expected one glossary:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
package glossary
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleGlossaryProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,11 +8,13 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -70,47 +72,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Transcript section:",
|
||||
"Protected glossary/context:",
|
||||
`"section_index": 0`,
|
||||
"punctuation, capitalization, spacing, and article cleanup only",
|
||||
"Do not make word substitutions",
|
||||
"Do not return speaker, start, or end fields",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(strings.ToLower(combined), "summarize") {
|
||||
t.Fatalf("prompt should not invite summarization")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Courtroom exchange with formal titles.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "must not override the transcript content") {
|
||||
t.Fatalf("expected no-override guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrammarModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -157,8 +118,12 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
module, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -185,9 +150,14 @@ func TestGrammarModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T) {
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "grammar:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleGrammarProposal {
|
||||
t.Fatalf("expected one grammar:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Hello, world" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package grammar
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to punctuation/capitalization/
|
||||
// spacing cleanup with strict meaning guards.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleGrammarProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleHomophonesProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -61,70 +63,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsContextAndConservativeConstraints(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Protected glossary/context:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"homophone correction assistant",
|
||||
"mistranscriptions of spoken English",
|
||||
"Do not add or remove punctuation",
|
||||
"Do not return speaker, start, or end fields",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"summarize the transcript",
|
||||
"style rewrite",
|
||||
"grammar cleanup",
|
||||
"punctuation cleanup",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(combined), forbidden) {
|
||||
t.Fatalf("prompt should not invite %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Tabletop session with fantasy names.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "must not override the transcript content") {
|
||||
t.Fatalf("expected no-override guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesOmitsTranscriptDescriptionSectionWhenEmpty(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, " ")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("did not expect empty transcript description section in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHomophonesModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -171,8 +109,12 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -199,9 +141,14 @@ func TestHomophonesModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "homophones:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleHomophonesProposal {
|
||||
t.Fatalf("expected one homophones:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "Jesters" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package homophones
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative homophone and
|
||||
// mistranscription updates.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleHomophonesProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
@@ -39,6 +39,6 @@ func (m *Module) Propose(ctx context.Context, req contracts.ProposalRequest) (co
|
||||
return proposal_generation.ExecuteModuleProposal(ctx, proposal_generation.ModuleProposalRequest{
|
||||
ProposalRequest: req,
|
||||
PromptID: prompts.PromptIDModuleSpokenWordProposal,
|
||||
BuildMessages: BuildProposalMessages,
|
||||
BuildVariables: proposal_generation.BuildProposalVariables,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
ck "gitea.maximumdirect.net/eric/ConfigKit"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposal_generation"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/validators"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
type fakeLLMClient struct {
|
||||
@@ -61,57 +63,6 @@ func tinyGlossary() *schema.Glossary {
|
||||
}}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesContainsContextAndMeaningGuardrails(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
if len(msgs) != 2 {
|
||||
t.Fatalf("expected 2 messages, got %d", len(msgs))
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
for _, want := range []string{
|
||||
"Protected glossary/context:",
|
||||
"Transcript section:",
|
||||
`"section_index": 0`,
|
||||
`"Aliases":`,
|
||||
`"Category": "faction"`,
|
||||
`"Summary": "Guild members"`,
|
||||
`"Plural": "Jesters"`,
|
||||
"conservative spoken-word cleanup assistant",
|
||||
"Preserve substantive meaning",
|
||||
"Do not paraphrase, summarize, reorder ideas",
|
||||
`"id": 1`,
|
||||
} {
|
||||
if !strings.Contains(combined, want) {
|
||||
t.Fatalf("expected prompt to contain %q", want)
|
||||
}
|
||||
}
|
||||
for _, forbidden := range []string{
|
||||
"style rewriting",
|
||||
"grammar-only cleanup",
|
||||
"punctuation-only cleanup",
|
||||
} {
|
||||
if strings.Contains(strings.ToLower(combined), forbidden) {
|
||||
t.Fatalf("prompt should not invite %q", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildProposalMessagesIncludesTranscriptDescriptionGuidance(t *testing.T) {
|
||||
msgs, err := BuildProposalMessages(tinyTranscript(), tinyGlossary(), 0, "Participants are discussing raid logistics.")
|
||||
if err != nil {
|
||||
t.Fatalf("BuildProposalMessages error: %v", err)
|
||||
}
|
||||
combined := msgs[0].Content + "\n" + msgs[1].Content
|
||||
if !strings.Contains(combined, "Transcript description (background context only):") {
|
||||
t.Fatalf("expected transcript description section in prompt")
|
||||
}
|
||||
if !strings.Contains(combined, "Do not invent corrections, facts, names, events, motivations, or speaker intent based on this description.") {
|
||||
t.Fatalf("expected anti-invention guidance in prompt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpokenWordModuleReplacementPolicy(t *testing.T) {
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
@@ -158,8 +109,12 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
t.Setenv("MODULE_SECRET", secret)
|
||||
loaded, loadErr := config.Load(ck.Bytes("config.yml", []byte("version: 2\nbackends:\n offline:\n endpoint: https://offline.invalid/v1\n credential: MODULE_SECRET\nllm:\n proposal_profile: base\n")), "")
|
||||
if loadErr != nil {
|
||||
t.Fatal(loadErr)
|
||||
}
|
||||
cfg := loaded.Config
|
||||
m, err := New()
|
||||
if err != nil {
|
||||
t.Fatalf("New error: %v", err)
|
||||
@@ -186,9 +141,14 @@ func TestSpokenWordModuleProposeMapsCorrectionsAndWritesDiagnostics(t *testing.T
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler run count 1, got %d", scheduler.runs)
|
||||
}
|
||||
if len(client.calls) != 1 || client.calls[0].StageName != "spoken_word:proposal" {
|
||||
if len(client.calls) != 1 || client.calls[0].PromptID != prompts.PromptIDModuleSpokenWordProposal {
|
||||
t.Fatalf("expected one spoken_word:proposal call, got %+v", client.calls)
|
||||
}
|
||||
call := client.calls[0]
|
||||
if call.Schema != "correction_set" || call.Context.Role != "proposal" || !strings.Contains(call.Variables["SectionJSON"], "section_index") || !strings.Contains(call.Variables["GlossaryJSON"], "Jesters") {
|
||||
t.Fatalf("proposal mapping incomplete: %+v", call)
|
||||
}
|
||||
|
||||
if len(out.Proposals) != 1 || out.Proposals[0].CorrectedText != "I think" {
|
||||
t.Fatalf("unexpected proposals: %+v", out)
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package spoken_word
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/promptcontext"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/prompts"
|
||||
)
|
||||
|
||||
// BuildProposalMessages constrains corrections to conservative dysfluency
|
||||
// cleanup with strict semantic preservation.
|
||||
func BuildProposalMessages(transcript *schema.Transcript, glossary *schema.Glossary, sectionIndex int, transcriptDescription string) ([]contracts.LLMMessage, error) {
|
||||
glossaryJSON, err := json.MarshalIndent(glossary, "", " ")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal glossary prompt context: %w", err)
|
||||
}
|
||||
|
||||
sectionJSON, err := promptcontext.MarshalTranscriptSectionJSON(transcript, sectionIndex)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal transcript prompt context: %w", err)
|
||||
}
|
||||
|
||||
system, user, _, err := prompts.RenderUserSystem(prompts.PromptIDModuleSpokenWordProposal, map[string]string{
|
||||
"TranscriptDescriptionBlock": promptcontext.TranscriptDescriptionBlock(transcriptDescription),
|
||||
"GlossaryJSON": string(glossaryJSON),
|
||||
"SectionJSON": string(sectionJSON),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return []contracts.LLMMessage{
|
||||
{Role: "system", Content: system},
|
||||
{Role: "user", Content: user},
|
||||
}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user