Complete Phase 11 proposal generation framework
This commit is contained in:
235
internal/framework/proposal_generation/generate.go
Normal file
235
internal/framework/proposal_generation/generate.go
Normal file
@@ -0,0 +1,235 @@
|
||||
// Package proposal_generation provides shared, deterministic LLM-backed
|
||||
// proposal-generation helpers. It only produces candidate proposals; validation
|
||||
// and application remain runner responsibilities.
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/llm"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
// InteractionDiagnosticsWriter writes machine-readable prompt/response artifacts.
|
||||
type InteractionDiagnosticsWriter interface {
|
||||
WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error)
|
||||
}
|
||||
|
||||
// InteractionArtifacts contains written diagnostics paths.
|
||||
type InteractionArtifacts struct {
|
||||
RequestMetadataPath string `json:"request_metadata_path,omitempty"`
|
||||
RequestPayloadPath string `json:"request_payload_path,omitempty"`
|
||||
ResponsePayloadPath string `json:"response_payload_path,omitempty"`
|
||||
ErrorPayloadPath string `json:"error_payload_path,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionProposal is one LLM response correction payload.
|
||||
type StructuredCorrectionProposal struct {
|
||||
TargetSegmentID int `json:"id"`
|
||||
OriginalText string `json:"original_text"`
|
||||
CorrectedText string `json:"corrected_text"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// StructuredCorrectionSet is the reusable structured LLM response model for
|
||||
// candidate correction proposals.
|
||||
type StructuredCorrectionSet struct {
|
||||
Corrections []StructuredCorrectionProposal `json:"corrections"`
|
||||
}
|
||||
|
||||
// Request captures reusable proposal-generation inputs for future modules.
|
||||
type Request struct {
|
||||
ModuleKey string `json:"module_key"`
|
||||
ModuleInstance string `json:"module_instance"`
|
||||
ReplacementPolicy proposals.ReplacementPolicy `json:"replacement_policy"`
|
||||
WorkingTranscript *schema.Transcript `json:"-"`
|
||||
Section *contracts.SectionMetadata `json:"section,omitempty"`
|
||||
Glossary *schema.Glossary `json:"-"`
|
||||
Config *config.Config `json:"-"`
|
||||
Messages []contracts.LLMMessage `json:"messages"`
|
||||
Model string `json:"model,omitempty"`
|
||||
StartIndex int `json:"start_index"`
|
||||
LLMClient contracts.StructuredLLMClient
|
||||
Scheduler contracts.LLMScheduler
|
||||
DiagnosticsDir string
|
||||
DiagnosticsWriter InteractionDiagnosticsWriter
|
||||
}
|
||||
|
||||
// Result contains generated candidate proposals and optional diagnostics paths.
|
||||
type Result struct {
|
||||
Corrections []proposals.CorrectionProposal `json:"corrections"`
|
||||
Enriched []proposals.EnrichedCorrectionProposal `json:"enriched"`
|
||||
Artifacts InteractionArtifacts `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateCandidates executes one structured LLM call and deterministically maps
|
||||
// its correction-set response into framework proposal types.
|
||||
func GenerateCandidates(ctx context.Context, req Request) (Result, error) {
|
||||
if strings.TrimSpace(req.ModuleKey) == "" {
|
||||
return Result{}, fmt.Errorf("module key must not be empty")
|
||||
}
|
||||
if strings.TrimSpace(req.ModuleInstance) == "" {
|
||||
return Result{}, fmt.Errorf("module instance must not be empty")
|
||||
}
|
||||
if req.StartIndex < 0 {
|
||||
return Result{}, fmt.Errorf("start index must be non-negative")
|
||||
}
|
||||
if req.LLMClient == nil {
|
||||
return Result{}, fmt.Errorf("structured LLM client is required")
|
||||
}
|
||||
if len(req.Messages) == 0 {
|
||||
return Result{}, fmt.Errorf("messages must not be empty")
|
||||
}
|
||||
|
||||
stage := buildStageName(req.ModuleInstance, req.Section)
|
||||
model := resolveModel(req.Config, req.Model)
|
||||
messages := append([]contracts.LLMMessage(nil), req.Messages...)
|
||||
|
||||
var writer InteractionDiagnosticsWriter
|
||||
if req.DiagnosticsWriter != nil {
|
||||
writer = req.DiagnosticsWriter
|
||||
} else if strings.TrimSpace(req.DiagnosticsDir) != "" {
|
||||
writer = diagnosticsWriterAdapter{
|
||||
writer: llm.NewDiagnosticsWriter(
|
||||
filepath.Join(req.DiagnosticsDir, req.ModuleInstance),
|
||||
proposalGenerationSecrets(req.Config),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
response StructuredCorrectionSet
|
||||
callErr error
|
||||
artifacts InteractionArtifacts
|
||||
)
|
||||
call := func(callCtx context.Context) error {
|
||||
_, callErr = req.LLMClient.CompleteStructured(callCtx, contracts.StructuredCompletionRequest{
|
||||
StageName: stage,
|
||||
Messages: messages,
|
||||
Model: model,
|
||||
}, &response)
|
||||
return callErr
|
||||
}
|
||||
if req.Scheduler != nil {
|
||||
callErr = req.Scheduler.Run(ctx, call)
|
||||
} else {
|
||||
callErr = call(ctx)
|
||||
}
|
||||
|
||||
if writer != nil {
|
||||
artifacts, _ = writer.WriteInteraction(
|
||||
stage,
|
||||
map[string]any{
|
||||
"module_key": req.ModuleKey,
|
||||
"module_instance": req.ModuleInstance,
|
||||
"replacement_policy": req.ReplacementPolicy,
|
||||
"section": req.Section,
|
||||
"start_index": req.StartIndex,
|
||||
"model": model,
|
||||
},
|
||||
map[string]any{
|
||||
"messages": messages,
|
||||
},
|
||||
response,
|
||||
errPayload(callErr),
|
||||
)
|
||||
}
|
||||
|
||||
if callErr != nil {
|
||||
return Result{}, fmt.Errorf("proposal generation completion failed: %w", callErr)
|
||||
}
|
||||
|
||||
corrections := make([]proposals.CorrectionProposal, 0, len(response.Corrections))
|
||||
enriched := make([]proposals.EnrichedCorrectionProposal, 0, len(response.Corrections))
|
||||
for i, raw := range response.Corrections {
|
||||
candidate := proposals.CorrectionProposal{
|
||||
TargetSegmentID: raw.TargetSegmentID,
|
||||
OriginalText: raw.OriginalText,
|
||||
CorrectedText: raw.CorrectedText,
|
||||
Confidence: raw.Confidence,
|
||||
}
|
||||
if err := candidate.Validate(); err != nil {
|
||||
return Result{}, fmt.Errorf("invalid structured correction at index %d: %w", i, err)
|
||||
}
|
||||
|
||||
corrections = append(corrections, candidate)
|
||||
enrichedCandidate := proposals.EnrichedCorrectionProposal{
|
||||
CorrectionProposal: candidate,
|
||||
ProposalMetadata: proposals.ProposalMetadata{
|
||||
ProposalIndex: req.StartIndex + i,
|
||||
ModuleKey: req.ModuleKey,
|
||||
ModuleInstance: req.ModuleInstance,
|
||||
},
|
||||
}
|
||||
if req.Section != nil {
|
||||
sectionIndex := req.Section.Index
|
||||
enrichedCandidate.SectionIndex = §ionIndex
|
||||
}
|
||||
enriched = append(enriched, enrichedCandidate)
|
||||
}
|
||||
|
||||
return Result{
|
||||
Corrections: corrections,
|
||||
Enriched: enriched,
|
||||
Artifacts: artifacts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildStageName(moduleInstance string, section *contracts.SectionMetadata) string {
|
||||
base := fmt.Sprintf("%s:proposal-generation", moduleInstance)
|
||||
if section == nil {
|
||||
return base
|
||||
}
|
||||
return fmt.Sprintf("%s:section-%04d", base, section.Index)
|
||||
}
|
||||
|
||||
func resolveModel(cfg *config.Config, override string) string {
|
||||
if strings.TrimSpace(override) != "" {
|
||||
return strings.TrimSpace(override)
|
||||
}
|
||||
if cfg == nil {
|
||||
return ""
|
||||
}
|
||||
return llm.ResolvePrimaryConfig(*cfg).Model
|
||||
}
|
||||
|
||||
func proposalGenerationSecrets(cfg *config.Config) []string {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
return []string{
|
||||
cfg.PrimaryLLM.APIKey,
|
||||
cfg.ValidationLLM.APIKey,
|
||||
cfg.EffectiveValidationLLMConfig().APIKey,
|
||||
}
|
||||
}
|
||||
|
||||
func errPayload(err error) any {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return map[string]any{"error": err.Error()}
|
||||
}
|
||||
|
||||
type diagnosticsWriterAdapter struct {
|
||||
writer *llm.DiagnosticsWriter
|
||||
}
|
||||
|
||||
func (a diagnosticsWriterAdapter) WriteInteraction(stage string, requestMetadata any, requestPayload any, responsePayload any, errorPayload any) (InteractionArtifacts, error) {
|
||||
artifacts, err := a.writer.WriteInteraction(stage, requestMetadata, requestPayload, responsePayload, errorPayload)
|
||||
if err != nil {
|
||||
return InteractionArtifacts{}, err
|
||||
}
|
||||
return InteractionArtifacts{
|
||||
RequestMetadataPath: artifacts.RequestMetadataPath,
|
||||
RequestPayloadPath: artifacts.RequestPayloadPath,
|
||||
ResponsePayloadPath: artifacts.ResponsePayloadPath,
|
||||
ErrorPayloadPath: artifacts.ErrorPayloadPath,
|
||||
}, nil
|
||||
}
|
||||
267
internal/framework/proposal_generation/generate_test.go
Normal file
267
internal/framework/proposal_generation/generate_test.go
Normal file
@@ -0,0 +1,267 @@
|
||||
package proposal_generation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/config"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
||||
"gitea.maximumdirect.net/eric/audita/internal/framework/proposals"
|
||||
)
|
||||
|
||||
type fakeStructuredClient struct {
|
||||
responses []StructuredCorrectionSet
|
||||
err error
|
||||
calls []contracts.StructuredCompletionRequest
|
||||
}
|
||||
|
||||
func (f *fakeStructuredClient) CompleteStructured(ctx context.Context, req contracts.StructuredCompletionRequest, out any) (contracts.StructuredCompletionResponse, error) {
|
||||
_ = ctx
|
||||
f.calls = append(f.calls, req)
|
||||
if f.err != nil {
|
||||
return contracts.StructuredCompletionResponse{}, f.err
|
||||
}
|
||||
target, ok := out.(*StructuredCorrectionSet)
|
||||
if !ok {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected output type")
|
||||
}
|
||||
if len(f.responses) == 0 {
|
||||
return contracts.StructuredCompletionResponse{}, errors.New("unexpected call")
|
||||
}
|
||||
*target = f.responses[0]
|
||||
f.responses = f.responses[1:]
|
||||
return contracts.StructuredCompletionResponse{}, nil
|
||||
}
|
||||
|
||||
type countingScheduler struct {
|
||||
runs int
|
||||
}
|
||||
|
||||
func (s *countingScheduler) Run(ctx context.Context, fn func(context.Context) error) error {
|
||||
s.runs++
|
||||
return fn(ctx)
|
||||
}
|
||||
|
||||
func defaultRequest(t *testing.T) Request {
|
||||
t.Helper()
|
||||
cfg := config.Default()
|
||||
return Request{
|
||||
ModuleKey: "test_module",
|
||||
ModuleInstance: "test_module",
|
||||
ReplacementPolicy: proposals.ReplacementPolicyRequireUnique,
|
||||
Config: &cfg,
|
||||
Messages: []contracts.LLMMessage{
|
||||
{Role: "system", Content: "system prompt"},
|
||||
{Role: "user", Content: "user prompt"},
|
||||
},
|
||||
StartIndex: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesSuccess(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 7, OriginalText: "teh", CorrectedText: "the", Confidence: 0.9},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
section := contracts.SectionMetadata{Index: 2}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.StartIndex = 10
|
||||
req.Section = §ion
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if len(got.Corrections) != 1 || len(got.Enriched) != 1 {
|
||||
t.Fatalf("unexpected proposal lengths: %+v", got)
|
||||
}
|
||||
if got.Enriched[0].ProposalIndex != 10 {
|
||||
t.Fatalf("expected proposal index 10, got %d", got.Enriched[0].ProposalIndex)
|
||||
}
|
||||
if got.Enriched[0].SectionIndex == nil || *got.Enriched[0].SectionIndex != 2 {
|
||||
t.Fatalf("expected section index 2, got %v", got.Enriched[0].SectionIndex)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMalformedStructuredResponse(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "", Confidence: 0.9},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "invalid structured correction") {
|
||||
t.Fatalf("expected structured response validation failure, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDeterministicIndexAssignment(t *testing.T) {
|
||||
baseResponse := StructuredCorrectionSet{
|
||||
Corrections: []StructuredCorrectionProposal{
|
||||
{TargetSegmentID: 1, OriginalText: "a", CorrectedText: "A", Confidence: 0.9},
|
||||
{TargetSegmentID: 2, OriginalText: "b", CorrectedText: "B", Confidence: 0.9},
|
||||
},
|
||||
}
|
||||
clientA := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
clientB := &fakeStructuredClient{responses: []StructuredCorrectionSet{baseResponse}}
|
||||
|
||||
reqA := defaultRequest(t)
|
||||
reqA.LLMClient = clientA
|
||||
reqA.StartIndex = 3
|
||||
first, err := GenerateCandidates(context.Background(), reqA)
|
||||
if err != nil {
|
||||
t.Fatalf("first generation failed: %v", err)
|
||||
}
|
||||
|
||||
reqB := defaultRequest(t)
|
||||
reqB.LLMClient = clientB
|
||||
reqB.StartIndex = 3
|
||||
second, err := GenerateCandidates(context.Background(), reqB)
|
||||
if err != nil {
|
||||
t.Fatalf("second generation failed: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(first.Enriched, second.Enriched) {
|
||||
t.Fatalf("expected stable enriched proposals\nfirst=%+v\nsecond=%+v", first.Enriched, second.Enriched)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesMultipleSectionsStableMetadata(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "alpha", CorrectedText: "ALPHA", Confidence: 0.9}}},
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 3, OriginalText: "charlie", CorrectedText: "CHARLIE", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
section0 := contracts.SectionMetadata{Index: 0}
|
||||
section1 := contracts.SectionMetadata{Index: 1}
|
||||
|
||||
req0 := defaultRequest(t)
|
||||
req0.LLMClient = client
|
||||
req0.Section = §ion0
|
||||
req0.StartIndex = 0
|
||||
part0, err := GenerateCandidates(context.Background(), req0)
|
||||
if err != nil {
|
||||
t.Fatalf("section 0 generation failed: %v", err)
|
||||
}
|
||||
|
||||
req1 := defaultRequest(t)
|
||||
req1.LLMClient = client
|
||||
req1.Section = §ion1
|
||||
req1.StartIndex = len(part0.Enriched)
|
||||
part1, err := GenerateCandidates(context.Background(), req1)
|
||||
if err != nil {
|
||||
t.Fatalf("section 1 generation failed: %v", err)
|
||||
}
|
||||
|
||||
all := append(append([]proposals.EnrichedCorrectionProposal(nil), part0.Enriched...), part1.Enriched...)
|
||||
if len(all) != 2 {
|
||||
t.Fatalf("expected 2 proposals, got %d", len(all))
|
||||
}
|
||||
if all[0].ProposalIndex != 0 || all[1].ProposalIndex != 1 {
|
||||
t.Fatalf("unexpected proposal indexes: %d, %d", all[0].ProposalIndex, all[1].ProposalIndex)
|
||||
}
|
||||
if all[0].SectionIndex == nil || *all[0].SectionIndex != 0 || all[1].SectionIndex == nil || *all[1].SectionIndex != 1 {
|
||||
t.Fatalf("unexpected section metadata: %+v", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesDiagnosticsWrittenAndRedacted(t *testing.T) {
|
||||
secret := "phase11-secret"
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: secret, CorrectedText: "safe", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
cfg := config.Default()
|
||||
cfg.PrimaryLLM.APIKey = secret
|
||||
req := defaultRequest(t)
|
||||
req.Config = &cfg
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
req.Messages = []contracts.LLMMessage{
|
||||
{Role: "system", Content: "include secret " + secret},
|
||||
{Role: "user", Content: "fix it"},
|
||||
}
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if got.Artifacts.ResponsePayloadPath == "" || got.Artifacts.RequestPayloadPath == "" {
|
||||
t.Fatalf("expected diagnostics artifact paths, got %+v", got.Artifacts)
|
||||
}
|
||||
|
||||
for _, path := range []string{got.Artifacts.RequestPayloadPath, got.Artifacts.ResponsePayloadPath} {
|
||||
raw, readErr := os.ReadFile(path)
|
||||
if readErr != nil {
|
||||
t.Fatalf("read artifact %q: %v", path, readErr)
|
||||
}
|
||||
if strings.Contains(string(raw), secret) {
|
||||
t.Fatalf("artifact leaked secret %q: %s", path, string(raw))
|
||||
}
|
||||
if !strings.Contains(string(raw), "[REDACTED]") {
|
||||
t.Fatalf("expected redaction marker in artifact %q: %s", path, string(raw))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesSchedulerUsage(t *testing.T) {
|
||||
client := &fakeStructuredClient{
|
||||
responses: []StructuredCorrectionSet{
|
||||
{Corrections: []StructuredCorrectionProposal{{TargetSegmentID: 1, OriginalText: "x", CorrectedText: "y", Confidence: 0.9}}},
|
||||
},
|
||||
}
|
||||
scheduler := &countingScheduler{}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.Scheduler = scheduler
|
||||
|
||||
_, err := GenerateCandidates(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateCandidates error: %v", err)
|
||||
}
|
||||
if scheduler.runs != 1 {
|
||||
t.Fatalf("expected scheduler to run once, got %d", scheduler.runs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateCandidatesClientError(t *testing.T) {
|
||||
client := &fakeStructuredClient{err: errors.New("boom")}
|
||||
req := defaultRequest(t)
|
||||
req.LLMClient = client
|
||||
req.DiagnosticsDir = t.TempDir()
|
||||
|
||||
got, err := GenerateCandidates(context.Background(), req)
|
||||
if err == nil || !strings.Contains(err.Error(), "completion failed") {
|
||||
t.Fatalf("expected completion failure, got %v", err)
|
||||
}
|
||||
if got.Artifacts.ResponsePayloadPath != "" {
|
||||
t.Fatalf("expected zero result on error, got %+v", got)
|
||||
}
|
||||
matches, globErr := filepath.Glob(filepath.Join(req.DiagnosticsDir, req.ModuleInstance, "*error-payload.json"))
|
||||
if globErr != nil {
|
||||
t.Fatalf("glob error: %v", globErr)
|
||||
}
|
||||
if len(matches) == 0 {
|
||||
t.Fatalf("expected error diagnostics artifact under %s", req.DiagnosticsDir)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user