Simplify references and output instructions in LLM prompts
This commit is contained in:
@@ -2,6 +2,7 @@ package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
@@ -17,7 +18,7 @@ import (
|
||||
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 7}
|
||||
@@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
|
||||
t.Fatalf("fatal operation lost: %+v, %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
|
||||
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
|
||||
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
|
||||
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
|
||||
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
|
||||
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
|
||||
}}}}
|
||||
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
|
||||
ProposalRequest: contracts.ProposalRequest{
|
||||
ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript},
|
||||
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
|
||||
LLMClient: client,
|
||||
},
|
||||
PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var payload struct {
|
||||
Segments []struct {
|
||||
ID int `json:"id"`
|
||||
Text string `json:"text"`
|
||||
} `json:"segments"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" {
|
||||
t.Fatalf("request references: %+v", payload)
|
||||
}
|
||||
if len(result.Proposals) != 4 {
|
||||
t.Fatalf("proposals: %+v", result)
|
||||
}
|
||||
for i, id := range []int{903, 71, 903, 0} {
|
||||
if result.Proposals[i].TargetSegmentID != id {
|
||||
t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i])
|
||||
}
|
||||
}
|
||||
if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 {
|
||||
t.Fatal("mapping changed deletion or source identities")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user