eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Simplify references and output instructions in LLM prompts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-25 12:38:23 +00:00
parent 325c4ea7d3
commit 011b021d47
31 changed files with 248 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
internal/framework/proposal_generation/adapter_integration_test.go
View File

@@ -61,7 +61,7 @@ func TestRepairedProposalStillFacesDomainChecks(t *testing.T) {
if err != nil {
t.Fatal(err)
}
generated, err := GenerateCandidates(context.Background(), Request{ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
generated, err := GenerateCandidates(context.Background(), Request{WorkingTranscript: transcript, ModuleKey: "grammar", ModuleInstance: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique, Config: &loaded.Config, PromptID: prompts.PromptIDModuleGrammarProposal, PromptVersion: prompts.VersionV1, Variables: vars, LLMClient: client})
if err != nil {
t.Fatal(err)
}

28
internal/framework/proposal_generation/generate.go
View File

@@ -36,10 +36,10 @@ type InteractionArtifacts struct {
// StructuredCorrectionProposal is one LLM response correction payload.
type StructuredCorrectionProposal struct {
TargetSegmentID int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
SegmentReference int `json:"id"`
OriginalText string `json:"original_text"`
CorrectedText string `json:"corrected_text"`
Confidence float64 `json:"confidence"`
}
// StructuredCorrectionSet is the reusable structured LLM response model for
@@ -142,6 +142,7 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
"replacement_policy": req.ReplacementPolicy,
"section": req.Section,
"start_index": req.StartIndex,
"segment_ids": segmentIDs(req.WorkingTranscript),
"prompt": completion.Prompt,
}
requestMetadata["response_schema"] = responseschema.MustLookup(responseschema.CorrectionSetKey).DiagnosticsMap()
@@ -185,8 +186,13 @@ func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
for i, raw := range response.Corrections {
// Unknown references remain invalid candidates for the existing domain validators.
targetID := 0
if req.WorkingTranscript != nil && raw.SegmentReference > 0 && raw.SegmentReference <= len(req.WorkingTranscript.Segments) {
targetID = req.WorkingTranscript.Segments[raw.SegmentReference-1].ID
}
candidate := proposals.CorrectionProposal{
TargetSegmentID: raw.TargetSegmentID,
TargetSegmentID: targetID,
OriginalText: raw.OriginalText,
CorrectedText: raw.CorrectedText,
Confidence: raw.Confidence,
@@ -247,3 +253,15 @@ func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata
ErrorPayloadPath: artifacts.ErrorPayloadPath,
}, nil
}
// segmentIDs records the request-local reference mapping in debug metadata.
func segmentIDs(transcript *schema.Transcript) []int {
if transcript == nil {
return nil
}
ids := make([]int, len(transcript.Segments))
for i, segment := range transcript.Segments {
ids[i] = segment.ID
}
return ids
}

28
internal/framework/proposal_generation/generate_test.go
View File

@@ -12,6 +12,7 @@ import (
ck "gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/audita/internal/core/config"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
"gitea.maximumdirect.net/eric/audita/internal/framework/outcomes"
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
@@ -77,6 +78,7 @@ func defaultRequest(t *testing.T) Request {
PromptID: "modules.grammar.proposal",
PromptVersion: "v1",
Variables: map[string]string{"SectionJSON": "[]"},
WorkingTranscript: &schema.Transcript{Segments: []schema.Segment{{ID: 7, Text: "teh"}, {ID: 25, Text: "other"}}},
StartIndex: 0,
}
}
@@ -86,7 +88,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
},
},
},
@@ -104,6 +106,9 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
t.Fatalf("unexpected proposal lengths: %+v", got)
}
if got.Corrections[0].TargetSegmentID != 7 {
t.Fatalf("local reference not resolved: %+v", got.Corrections)
}
if got.Enriched[0].ProposalIndex != 10 {
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
}
@@ -118,7 +123,7 @@ func TestGenerateCandidatesSuccess(t *testing.T) {
func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
req := defaultRequest(t)
@@ -140,7 +145,7 @@ func TestGenerateCandidatesUsesCorrectionSetSchema(t *testing.T) {
func TestGenerateCandidatesDiagnosticsIncludeSchemaMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
diag := &captureDiagnosticsWriter{}
@@ -172,7 +177,7 @@ func TestGenerateCandidatesInvalidCorrectionIsPreservedForLaterValidation(t *tes
responses: []StructuredCorrectionSet{
{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
{SegmentReference: 0, OriginalText: "x", CorrectedText: "", Confidence: 1.2},
},
},
},
@@ -235,8 +240,8 @@ func TestGenerateCandidatesValidEmptyResponseIsSuccessful(t *testing.T) {
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
baseResponse := StructuredCorrectionSet{
Corrections: []StructuredCorrectionProposal{
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
{SegmentReference: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
{SegmentReference: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
},
}
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
@@ -266,8 +271,8 @@ func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
},
}
section0 := contracts.SectionMetadata{Index: 0}
@@ -276,6 +281,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req0 := defaultRequest(t)
req0.LLMClient = client
req0.Section = &section0
req0.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "alpha"}}}
req0.StartIndex = 0
part0, err := GenerateCandidates(context.Background(), req0)
if err != nil {
@@ -285,6 +291,7 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
req1 := defaultRequest(t)
req1.LLMClient = client
req1.Section = &section1
req1.WorkingTranscript = &schema.Transcript{Segments: []schema.Segment{{ID: 903, Text: "charlie"}}}
req1.StartIndex = len(part0.Enriched)
part1, err := GenerateCandidates(context.Background(), req1)
if err != nil {
@@ -295,6 +302,9 @@ func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
if len(all) != 2 {
t.Fatalf("expected 2 proposals, got %d", len(all))
}
if all[0].TargetSegmentID != 71 || all[1].TargetSegmentID != 903 {
t.Fatalf("section references did not reset: %+v", all)
}
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
}
@@ -308,7 +318,7 @@ func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
validationSecret := "proposal-validation-secret"
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: validationSecret, CorrectedText: "safe", Confidence: 0.9}}},
},
}
t.Setenv("PRIMARY_SECRET", primarySecret)

47
internal/framework/proposal_generation/module_proposal_test.go
View File

@@ -2,6 +2,7 @@ package proposal_generation
import (
"context"
"encoding/json"
"errors"
"testing"
@@ -17,7 +18,7 @@ import (
func TestExecuteModuleProposalBuildsMessagesFromSectionAndDescription(t *testing.T) {
client := &fakeStructuredClient{
responses: []StructuredCorrectionSet{
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
{Corrections: []StructuredCorrectionProposal{{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9}}},
},
}
section := contracts.SectionMetadata{Index: 7}
@@ -135,3 +136,47 @@ func TestExecuteModuleProposalPreservesFatalOperation(t *testing.T) {
t.Fatalf("fatal operation lost: %+v, %v", result, err)
}
}
func TestModuleProposalMapsLocalReferencesWithoutGuessing(t *testing.T) {
transcript := &schema.Transcript{Segments: []schema.Segment{{ID: 71, Text: "teh cat"}, {ID: 903, Text: "uh hello"}}}
client := &fakeStructuredClient{responses: []StructuredCorrectionSet{{Corrections: []StructuredCorrectionProposal{
{SegmentReference: 2, OriginalText: "uh ", CorrectedText: "", Confidence: .9},
{SegmentReference: 1, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
{SegmentReference: 2, OriginalText: "hello", CorrectedText: "Hello", Confidence: .9},
{SegmentReference: 71, OriginalText: "teh", CorrectedText: "the", Confidence: .9},
}}}}
result, err := ExecuteModuleProposal(context.Background(), ModuleProposalRequest{
ProposalRequest: contracts.ProposalRequest{
ExecutionContext: contracts.ExecutionContext{WorkingTranscript: transcript},
RunSpec: contracts.ModuleRunSpec{ModuleKey: "grammar", InstanceName: "grammar", ReplacementPolicy: proposals.ReplacementPolicyRequireUnique},
LLMClient: client,
},
PromptID: prompts.PromptIDModuleGrammarProposal, BuildVariables: BuildProposalVariables,
})
if err != nil {
t.Fatal(err)
}
var payload struct {
Segments []struct {
ID int `json:"id"`
Text string `json:"text"`
} `json:"segments"`
}
if err := json.Unmarshal([]byte(client.calls[0].Variables["SectionJSON"]), &payload); err != nil {
t.Fatal(err)
}
if len(payload.Segments) != 2 || payload.Segments[0].ID != 1 || payload.Segments[1].ID != 2 || payload.Segments[1].Text != "uh hello" {
t.Fatalf("request references: %+v", payload)
}
if len(result.Proposals) != 4 {
t.Fatalf("proposals: %+v", result)
}
for i, id := range []int{903, 71, 903, 0} {
if result.Proposals[i].TargetSegmentID != id {
t.Fatalf("mapped proposal %d: %+v", i, result.Proposals[i])
}
}
if result.Proposals[0].CorrectedText != "" || transcript.Segments[0].ID != 71 || transcript.Segments[1].ID != 903 {
t.Fatal("mapping changed deletion or source identities")
}
}